host
cd33a929ca
Policy carries an operator-declared model_capabilities directory and per-tier requires (tool_calling/reasoning/vision booleans, min_context_tokens). decide_routing checks the policy tier model in active mode: capable applies, incapable falls back to a capable worker default, and if both fail or are unknown it returns capability_blocked, which the worker turns into a failed task (v3 tools-gate shape, runner never called). Shadow mode only annotates the reason, never blocks. Explicit task and worker CLI models bypass the check by design. Invalid requires drops only that tier (fail-closed, like invalid tier mode). Dry-run reports per-tier capability status and blocks on active tiers that fail or cannot be verified.
329 lines
14 KiB
Python
329 lines
14 KiB
Python
"""Routing policy dry-run (v0d): validate before applying, never apply.
|
|
|
|
A routing policy takes effect the moment a worker loads it from the
|
|
host env -- there is no CI gate for an env change. A typo in a tier
|
|
mode silently drops that tier, a bad model name is never checked
|
|
against a whitelist (models are not whitelisted by design), and a
|
|
policy file that fails to parse is silently ignored by the loader
|
|
(fail-closed to the worker default, but also silently). This module
|
|
surfaces all of that BEFORE the policy is applied, and it never
|
|
applies anything itself: it only reports what WOULD happen if the
|
|
given policy document were loaded.
|
|
|
|
What it reports, per tier (plan / goal / default):
|
|
* the effective model and mode, and whether the mode came from an
|
|
explicit tier mode or from the global mode,
|
|
* tiers that were dropped, and why (invalid tier mode, no model,
|
|
unrecognised entry) -- a dropped tier is a BLOCKER, not a warning
|
|
to skim past, because applying the policy would silently not route
|
|
that tier,
|
|
* tiers absent from the document (worker default, not routed),
|
|
* unknown tier names in the document (ignored by the parser),
|
|
* whether any tier would actually activate on next dispatch,
|
|
* capability checks (v8): per tier, whether its model meets
|
|
its requires against the policy's model_capabilities
|
|
directory -- an active tier that fails or cannot be verified
|
|
is a BLOCKER (it would block dispatch at the worker
|
|
capability gate); a shadow/off tier with the same problem
|
|
is a warning only.
|
|
|
|
What it does NOT do, honestly:
|
|
* It does NOT validate model names against a provider or relay
|
|
catalog. Task models are deliberately not whitelisted in this
|
|
codebase (the relay catalog changes), so a model typo here is
|
|
reported as "unverified, not whitelisted" only if the caller
|
|
passes a known-model set; without one, model names are reported
|
|
as-is and the caller must check them against the relay catalog
|
|
themselves. This module never invents a catalog.
|
|
* It does NOT check shadow sample counts. That is the shadow
|
|
report's job (GET /routing-shadow); a dry-run that said "ready
|
|
to activate" without sample data would be fabrication.
|
|
* It does NOT change host env, worker state, or any file. The CLI
|
|
exits 1 when the policy is invalid or any tier was dropped or
|
|
unknown, so a deploy script can gate on it; warnings alone
|
|
(no version, fallback_models present) do not block, but are
|
|
always printed.
|
|
|
|
Loading order, mirrored from load_routing_policy in models.py:
|
|
file (CLOUD_ROUTING_POLICY_FILE) first, then JSON
|
|
(CLOUD_ROUTING_POLICY_JSON) overrides whole tier entries (not
|
|
per-field: a JSON tier entry replaces the file's tier entry
|
|
entirely), then CLOUD_ROUTING_MODE overrides the GLOBAL mode
|
|
only -- it never overrides an explicit tier mode. Failed file/JSON
|
|
sources, which the loader silently ignores, are reported here as
|
|
warnings instead of being swallowed.
|
|
"""
|
|
|
|
import argparse
|
|
import json
|
|
import os
|
|
import sys
|
|
|
|
try:
|
|
from cloud.models import ROUTING_MODES, ROUTING_TIERS
|
|
from cloud.models import parse_routing_policy
|
|
from cloud.models import parse_tier_requires
|
|
from cloud.models import check_model_capabilities
|
|
except ImportError: # run as a script from the repo root
|
|
sys.path.insert(0, os.path.join(
|
|
os.path.dirname(os.path.abspath(__file__)), ".."))
|
|
from cloud.models import ROUTING_MODES, ROUTING_TIERS
|
|
from cloud.models import parse_routing_policy
|
|
from cloud.models import parse_tier_requires
|
|
from cloud.models import check_model_capabilities
|
|
|
|
|
|
def _tier_dropped_reason(raw_entry):
|
|
if isinstance(raw_entry, dict):
|
|
if "mode" in raw_entry and \
|
|
raw_entry.get("mode") not in ROUTING_MODES:
|
|
return "invalid tier mode %r" % (raw_entry.get("mode"),)
|
|
if "requires" in raw_entry and \
|
|
parse_tier_requires(raw_entry.get("requires")) is None:
|
|
return "invalid tier requires %r" % (raw_entry.get("requires"),)
|
|
return "no usable model"
|
|
if isinstance(raw_entry, str):
|
|
return "empty model" if not raw_entry.strip() else "unknown"
|
|
return "unrecognised tier entry (expected string or dict)"
|
|
|
|
|
|
def dry_run_policy_document(data, env_mode_override=None):
|
|
"""Dry-run one merged policy document (pure, no IO).
|
|
|
|
``data`` is the merged document as parse_routing_policy would
|
|
receive it. ``env_mode_override`` is the raw CLOUD_ROUTING_MODE
|
|
value (or None): a valid value overrides the global mode only,
|
|
an invalid value is ignored with a warning.
|
|
"""
|
|
warnings = []
|
|
tiers_report = {}
|
|
dropped = []
|
|
unknown = []
|
|
if not isinstance(data, dict):
|
|
return {
|
|
"valid": False, "blocked": True, "version": None,
|
|
"global_mode": None, "global_mode_source": None,
|
|
"tiers": tiers_report, "dropped_tiers": dropped,
|
|
"unknown_tiers": unknown,
|
|
"warnings": ["policy document is not an object; "
|
|
"the loader would ignore it entirely"],
|
|
"would_activate": [], "would_shadow": [], "would_off": [],
|
|
}
|
|
raw_tiers = data.get("tiers")
|
|
if isinstance(raw_tiers, dict):
|
|
for name in sorted(raw_tiers):
|
|
if name not in ROUTING_TIERS:
|
|
unknown.append(name)
|
|
warnings.append(
|
|
"unknown tier %r is ignored by the parser" % name)
|
|
policy = parse_routing_policy(data)
|
|
if policy is None:
|
|
warnings.append(
|
|
"policy is unusable (no usable tiers, or global mode "
|
|
"outside off/shadow/active); the loader would ignore it "
|
|
"entirely and workers would keep the worker default")
|
|
return {
|
|
"valid": False, "blocked": True, "version": None,
|
|
"global_mode": None, "global_mode_source": None,
|
|
"tiers": tiers_report, "dropped_tiers": dropped,
|
|
"unknown_tiers": unknown, "warnings": warnings,
|
|
"would_activate": [], "would_shadow": [], "would_off": [],
|
|
}
|
|
global_mode = policy.mode
|
|
global_source = "policy"
|
|
if env_mode_override:
|
|
if env_mode_override in ROUTING_MODES:
|
|
global_mode = env_mode_override
|
|
global_source = "env_override"
|
|
warnings.append(
|
|
"CLOUD_ROUTING_MODE overrides the global mode to %r, "
|
|
"but does NOT override explicit tier modes" %
|
|
env_mode_override)
|
|
else:
|
|
warnings.append(
|
|
"CLOUD_ROUTING_MODE %r is invalid and ignored; "
|
|
"the global mode from the policy stands" %
|
|
env_mode_override)
|
|
# Apply the env global override to the effective per-tier modes
|
|
# without mutating the parsed policy: a tier with an explicit
|
|
# mode keeps it, a tier without one follows the overridden global.
|
|
effective_policy_mode = global_mode
|
|
cap_blocked = False
|
|
for tier in ROUTING_TIERS:
|
|
raw_entry = raw_tiers.get(tier) if isinstance(raw_tiers,
|
|
dict) else None
|
|
if tier not in policy.tiers:
|
|
if raw_entry is not None:
|
|
reason = _tier_dropped_reason(raw_entry)
|
|
dropped.append(tier)
|
|
warnings.append(
|
|
"tier %r was dropped: %s; applying this policy "
|
|
"would silently not route %r tasks" %
|
|
(tier, reason, tier))
|
|
tiers_report[tier] = {
|
|
"model": None, "mode": None,
|
|
"mode_source": None, "status": "dropped",
|
|
"dropped_reason": reason,
|
|
}
|
|
else:
|
|
tiers_report[tier] = {
|
|
"model": None, "mode": None,
|
|
"mode_source": None, "status": "absent",
|
|
"dropped_reason": None,
|
|
}
|
|
continue
|
|
if tier in policy.tier_modes:
|
|
mode = policy.tier_modes[tier]
|
|
mode_source = "tier_mode"
|
|
else:
|
|
mode = effective_policy_mode
|
|
mode_source = "global_mode"
|
|
status = "active" if mode == "active" else (
|
|
"shadow" if mode == "shadow" else "off")
|
|
# Capability check (v8): the same check decide_routing
|
|
# runs, reported before activation. "not_applicable"
|
|
# when the tier carries no requirements; "unknown" when
|
|
# requirements exist but the model (or a needed field)
|
|
# is absent from the directory.
|
|
requires = policy.tier_requirements.get(tier) or {}
|
|
if not requires:
|
|
cap = {"required": None, "missing": [], "unknown": False,
|
|
"status": "not_applicable"}
|
|
else:
|
|
cap_ok, cap_missing, cap_unknown = check_model_capabilities(
|
|
policy.model_capabilities, policy.tiers[tier], requires)
|
|
cap = {"required": requires, "missing": cap_missing,
|
|
"unknown": cap_unknown,
|
|
"status": "ok" if cap_ok else (
|
|
"unknown" if cap_unknown else "fail")}
|
|
tiers_report[tier] = {
|
|
"model": policy.tiers[tier], "mode": mode,
|
|
"mode_source": mode_source, "status": status,
|
|
"dropped_reason": None, "capabilities": cap,
|
|
}
|
|
if cap["status"] in ("fail", "unknown"):
|
|
note = ("tier %r model %r %s capability: %s" % (
|
|
tier, policy.tiers[tier],
|
|
"cannot verify" if cap["status"] == "unknown"
|
|
else "lacks",
|
|
", ".join(cap["missing"])))
|
|
if status == "active":
|
|
# An active tier that fails/unknowns its capability
|
|
# check would block dispatch (worker capability
|
|
# gate), so it blocks the dry-run too.
|
|
cap_blocked = True
|
|
warnings.append(note + "; applying this policy "
|
|
"would block %r tasks at dispatch" % tier)
|
|
else:
|
|
warnings.append(note + "; %s mode does not block "
|
|
"dispatch" % status)
|
|
would_activate = [t for t in ROUTING_TIERS
|
|
if tiers_report[t]["status"] == "active"]
|
|
would_shadow = [t for t in ROUTING_TIERS
|
|
if tiers_report[t]["status"] == "shadow"]
|
|
would_off = [t for t in ROUTING_TIERS
|
|
if tiers_report[t]["status"] == "off"]
|
|
if policy.version is None:
|
|
warnings.append(
|
|
"no policy version: policy changes will be hard to "
|
|
"attribute in shadow reports (decisions record "
|
|
"policy_version)")
|
|
if data.get("fallback_models") is not None:
|
|
warnings.append(
|
|
"fallback_models in the policy are parsed but NOT "
|
|
"applied; only worker CLI --fallback-models enables a "
|
|
"real fallback retry")
|
|
if would_activate:
|
|
warnings.append(
|
|
"activation warning: tier(s) %s would actually route "
|
|
"tasks on next worker dispatch if this policy were "
|
|
"applied; this dry-run does not activate anything" %
|
|
", ".join(would_activate))
|
|
blocked = bool(dropped or unknown or cap_blocked)
|
|
return {
|
|
"valid": True, "blocked": blocked,
|
|
"version": policy.version, "global_mode": global_mode,
|
|
"global_mode_source": global_source,
|
|
"tiers": tiers_report, "dropped_tiers": dropped,
|
|
"unknown_tiers": unknown, "warnings": warnings,
|
|
"would_activate": would_activate,
|
|
"would_shadow": would_shadow, "would_off": would_off,
|
|
}
|
|
|
|
|
|
def dry_run_from_env(env=None):
|
|
"""Dry-run from an env mapping, mirroring load_routing_policy.
|
|
|
|
Returns the report dict. File/JSON failures that the loader
|
|
silently ignores are reported as warnings here.
|
|
"""
|
|
env = env if env is not None else os.environ
|
|
warnings = []
|
|
merged = None
|
|
path = env.get("CLOUD_ROUTING_POLICY_FILE") or ""
|
|
if path:
|
|
try:
|
|
with open(path, encoding="utf-8") as fh:
|
|
merged = json.load(fh)
|
|
except (OSError, ValueError) as exc:
|
|
warnings.append(
|
|
"policy file %r could not be read/parsed (%s); "
|
|
"the loader would ignore it" % (path, exc))
|
|
merged = None
|
|
raw = env.get("CLOUD_ROUTING_POLICY_JSON") or ""
|
|
if raw:
|
|
try:
|
|
override = json.loads(raw)
|
|
except ValueError as exc:
|
|
warnings.append(
|
|
"CLOUD_ROUTING_POLICY_JSON could not be parsed "
|
|
"(%s); the loader would ignore it" % exc)
|
|
override = None
|
|
if isinstance(override, dict):
|
|
if not isinstance(merged, dict):
|
|
merged = override
|
|
else:
|
|
merged = dict(merged)
|
|
merged.update(override)
|
|
if isinstance(merged.get("tiers"), dict) and \
|
|
isinstance(override.get("tiers"), dict):
|
|
tiers = dict(merged["tiers"])
|
|
tiers.update(override["tiers"])
|
|
merged["tiers"] = tiers
|
|
report = dry_run_policy_document(
|
|
merged, env_mode_override=env.get("CLOUD_ROUTING_MODE") or None)
|
|
# Source-failure warnings belong before the document warnings,
|
|
# and they do not change blocked (a failed source alone is not
|
|
# a dropped tier; the document report already covers validity).
|
|
report["warnings"] = warnings + report["warnings"]
|
|
return report
|
|
|
|
|
|
def main(argv=None):
|
|
parser = argparse.ArgumentParser(
|
|
description="Dry-run a routing policy (never applies it).")
|
|
parser.add_argument("--policy-file", default=None,
|
|
help="Policy JSON file (same as "
|
|
"CLOUD_ROUTING_POLICY_FILE)")
|
|
parser.add_argument("--policy-json", default=None,
|
|
help="Policy JSON string (same as "
|
|
"CLOUD_ROUTING_POLICY_JSON)")
|
|
parser.add_argument("--mode", default=None,
|
|
help="Global mode override (same as "
|
|
"CLOUD_ROUTING_MODE)")
|
|
args = parser.parse_args(argv)
|
|
env = {}
|
|
if args.policy_file:
|
|
env["CLOUD_ROUTING_POLICY_FILE"] = args.policy_file
|
|
if args.policy_json:
|
|
env["CLOUD_ROUTING_POLICY_JSON"] = args.policy_json
|
|
if args.mode:
|
|
env["CLOUD_ROUTING_MODE"] = args.mode
|
|
report = dry_run_from_env(env)
|
|
print(json.dumps(report, indent=2, sort_keys=True))
|
|
return 1 if (not report["valid"] or report["blocked"]) else 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|