refactor: audit batch 3 (I-3 capability verdict, I-4 policy merge)
- I-3: new models.evaluate_capability returning a CapabilityVerdict (ok/missing/unknown from check, known/unknown_missing from split, status/verb derivations); decide_routing, worker gateway check, worker persisted recheck, dry-run tier+gateway checks and server shadow endpoint all call it. Blocking rules stay at the call sites (gateway has no worker-default fallback, recheck blocks only on known, dry-run/shadow never block); all user-visible audit details, dry-run notes and shadow notes are byte-identical. - I-4: new models.merge_policy_documents shared by load_routing_policy and dry_run_from_env; the old tiers-update block is kept verbatim (it was a no-op after update(): JSON tiers replace the whole map, file tiers absent from JSON are dropped -- preserved, not fixed, as that would be a behaviour change). dropped_tiers/partial_failures remain load-time records, not merge products. - Tests: test_audit_batch3.py, 13 tests (verdict matrix + parity with check/split, decide/dry-run wording, merge boundaries + 300 seeded random pairs + loader/dry-run agreement). Full suite: 1694 OK (skipped=11), baseline 1681 + 13, 0 failures.
This commit is contained in:
1 parent
a7f77c8a98
commit
06e4e16bbb
5 files changed
+485
-68
No files matched your search
+117
-17
@@ -289,6 +289,115 @@ def split_model_capability_missing(capabilities, model, requires):
|
||||
return sorted(known), sorted(unknown)
|
||||
|
||||
|
||||
class CapabilityVerdict:
|
||||
"""Structured result of one capability check (audit batch 3, I-3).
|
||||
|
||||
One call replaces the previous check+split pair at the five
|
||||
capability sites (decide_routing, worker gateway check, worker
|
||||
persisted-model recheck, dry-run tier/gateway checks, server
|
||||
shadow endpoint). The fields are exactly what those sites used
|
||||
before, so no site changes behaviour by adopting it:
|
||||
|
||||
- ``ok`` / ``missing`` / ``unknown`` are the total-check fields
|
||||
from check_model_capabilities (decide/dry-run/shadow wording);
|
||||
- ``known_missing`` / ``unknown_missing`` are the split fields
|
||||
from split_model_capability_missing (gateway explicit split,
|
||||
persisted recheck);
|
||||
- ``status`` is "ok", or "unknown" when not ok and any part is
|
||||
unverifiable, else "fail" (the dry-run/shadow status rule);
|
||||
- ``verb`` is "cannot verify" when unknown else "lacks" (the
|
||||
dry-run/gateway note wording); it is only meaningful when
|
||||
not ok.
|
||||
|
||||
Edge preserved, not fixed: for a model absent from the
|
||||
directory, ``missing`` lists every requires key (the total
|
||||
check's rule) while ``unknown_missing`` lists only recognised
|
||||
requirement keys (the split's rule). Callers that previously
|
||||
used one function keep getting that function's values; the
|
||||
blocking rules themselves stay at the call sites (gateway has
|
||||
no worker-default fallback, recheck blocks only on known,
|
||||
dry-run/shadow never block), they are not folded into this
|
||||
verdict.
|
||||
"""
|
||||
|
||||
def __init__(self, ok, missing, unknown, known_missing,
|
||||
unknown_missing):
|
||||
self.ok = bool(ok)
|
||||
self.missing = list(missing)
|
||||
self.unknown = bool(unknown)
|
||||
self.known_missing = list(known_missing)
|
||||
self.unknown_missing = list(unknown_missing)
|
||||
|
||||
@property
|
||||
def status(self):
|
||||
if self.ok:
|
||||
return "ok"
|
||||
return "unknown" if self.unknown else "fail"
|
||||
|
||||
@property
|
||||
def verb(self):
|
||||
return "cannot verify" if self.unknown else "lacks"
|
||||
|
||||
|
||||
def evaluate_capability(capabilities, model, requires):
|
||||
"""Check one model against requires, returning a CapabilityVerdict.
|
||||
|
||||
Pure: it only calls check_model_capabilities and
|
||||
split_model_capability_missing, so its fields are definitionally
|
||||
identical to what the five call sites computed before with two
|
||||
calls. ``requires`` is expected to be already cleaned by
|
||||
parse_tier_requires (all five sites pass tier_requirements);
|
||||
uncleaned keys behave exactly as the two underlying functions
|
||||
behave (see CapabilityVerdict).
|
||||
"""
|
||||
ok, missing, unknown = check_model_capabilities(
|
||||
capabilities, model, requires)
|
||||
known, unknown_missing = split_model_capability_missing(
|
||||
capabilities, model, requires)
|
||||
return CapabilityVerdict(ok, missing, unknown, known,
|
||||
unknown_missing)
|
||||
|
||||
|
||||
def merge_policy_documents(base, override):
|
||||
"""Merge a file policy document with a JSON override document.
|
||||
|
||||
This is the merge load_routing_policy and dry_run_from_env
|
||||
always shared line-for-line, now in one place (audit batch 3,
|
||||
I-4). ``base`` is the file document (or None when unread/absent),
|
||||
``override`` is the JSON document (or None when absent/unparsed).
|
||||
Rules, exactly as before:
|
||||
|
||||
- override not a dict: base unchanged;
|
||||
- base not a dict: override;
|
||||
- otherwise: a shallow copy of base updated with override, so
|
||||
JSON replaces whole keys — including the whole "tiers" map.
|
||||
Per-tier merging does NOT happen: the old tiers-update block
|
||||
after update() copied override's tiers back onto themselves
|
||||
(merged["tiers"] already WAS override's tiers), i.e. it was
|
||||
a no-op, and it is kept verbatim below so this function stays
|
||||
a line-for-line move, not a semantic "fix". File tiers absent
|
||||
from the JSON document are therefore dropped, not preserved;
|
||||
changing that would be a behaviour change for another batch.
|
||||
|
||||
Source-level failure records (dropped_tiers/partial_failures
|
||||
from batch 2) are NOT a product of this merge: they are filled
|
||||
in by parse/load afterwards, and this function neither reads
|
||||
nor writes them.
|
||||
"""
|
||||
if not isinstance(override, dict):
|
||||
return base
|
||||
if not isinstance(base, dict):
|
||||
return override
|
||||
merged = dict(base)
|
||||
merged.update(override)
|
||||
if isinstance(merged.get("tiers"), dict) and \
|
||||
isinstance(override.get("tiers"), dict):
|
||||
tiers = dict(merged["tiers"])
|
||||
tiers.update(override["tiers"])
|
||||
merged["tiers"] = tiers
|
||||
return merged
|
||||
|
||||
|
||||
def prompt_len_bucket(prompt) -> str:
|
||||
"""Bucket a prompt by length for logging only, never for routing.
|
||||
|
||||
@@ -578,16 +687,7 @@ def load_routing_policy(env=None):
|
||||
failures.append("CLOUD_ROUTING_POLICY_JSON: %s" % exc)
|
||||
override = None
|
||||
if isinstance(override, dict):
|
||||
if not isinstance(merged, dict):
|
||||
merged = override
|
||||
else:
|
||||
merged = dict(merged)
|
||||
merged.update(override)
|
||||
if isinstance(merged.get("tiers"), dict) and \
|
||||
isinstance(override.get("tiers"), dict):
|
||||
tiers = dict(merged["tiers"])
|
||||
tiers.update(override["tiers"])
|
||||
merged["tiers"] = tiers
|
||||
merged = merge_policy_documents(merged, override)
|
||||
policy = parse_routing_policy(merged)
|
||||
if policy is None:
|
||||
if configured:
|
||||
@@ -741,13 +841,13 @@ def decide_routing(task, worker_model, policy=None,
|
||||
policy_version=policy.version, applied=True,
|
||||
reason="%s tier mode is off" % tier, signals=signals)
|
||||
requires = policy.tier_requirements.get(tier) or {}
|
||||
cap_ok, cap_missing, cap_unknown = check_model_capabilities(
|
||||
cap = evaluate_capability(
|
||||
policy.model_capabilities, tier_model, requires)
|
||||
cap_note = ""
|
||||
if requires and not cap_ok:
|
||||
if requires and not cap.ok:
|
||||
cap_note = " (capability %s: %s)" % (
|
||||
"unknown" if cap_unknown else "lacks",
|
||||
", ".join(cap_missing))
|
||||
"unknown" if cap.unknown else "lacks",
|
||||
", ".join(cap.missing))
|
||||
if effective_mode == "shadow":
|
||||
return RoutingDecision(
|
||||
task_id=task.get("id"), tier=tier, chosen_model=worker_model,
|
||||
@@ -756,7 +856,7 @@ def decide_routing(task, worker_model, policy=None,
|
||||
reason="%s tier suggests %s (shadow, not applied)%s"
|
||||
% (tier, tier_model, cap_note), signals=signals)
|
||||
# effective_mode == "active"
|
||||
if cap_ok:
|
||||
if cap.ok:
|
||||
return RoutingDecision(
|
||||
task_id=task.get("id"), tier=tier, chosen_model=tier_model,
|
||||
suggested_model=None, source="policy",
|
||||
@@ -767,9 +867,9 @@ def decide_routing(task, worker_model, policy=None,
|
||||
# default against the same requirements before blocking:
|
||||
# a capable worker default keeps the task moving without
|
||||
# violating the requirement.
|
||||
worker_ok, _w_missing, _w_unknown = check_model_capabilities(
|
||||
worker_cap = evaluate_capability(
|
||||
policy.model_capabilities, worker_model, requires)
|
||||
if worker_ok:
|
||||
if worker_cap.ok:
|
||||
return RoutingDecision(
|
||||
task_id=task.get("id"), tier=tier, chosen_model=worker_model,
|
||||
suggested_model=None, source="worker_default",
|
||||
|
||||
Reference in new issue
Block a user