host
8fe4ad2f38
New module cloud/routing_dryrun.py: dry-run a merged policy document or an env mapping (file first, JSON overrides whole tier entries, CLOUD_ROUTING_MODE overrides global mode only, never explicit tier modes), reporting per-tier effective model/mode/mode-source, dropped tiers with reasons, absent and unknown tiers, and would_activate/would_shadow/would_off. Dropped or unknown tiers block (CLI exit 1); warnings alone (no version, fallback_models parsed but not applied, failed file/JSON sources that the loader silently ignores) do not block but are always printed. Model names are NOT validated against a catalog (deliberately not whitelisted) and sample counts are NOT checked here (that is the shadow report's job); the CLI never writes env, files, or worker state. Tests: test_routing_dryrun.py (8 tests).
283 lines
12 KiB
Python
283 lines
12 KiB
Python
"""Routing policy dry-run (v0d): validate before applying, never apply.
|
|
|
|
A routing policy takes effect the moment a worker loads it from the
|
|
host env -- there is no CI gate for an env change. A typo in a tier
|
|
mode silently drops that tier, a bad model name is never checked
|
|
against a whitelist (models are not whitelisted by design), and a
|
|
policy file that fails to parse is silently ignored by the loader
|
|
(fail-closed to the worker default, but also silently). This module
|
|
surfaces all of that BEFORE the policy is applied, and it never
|
|
applies anything itself: it only reports what WOULD happen if the
|
|
given policy document were loaded.
|
|
|
|
What it reports, per tier (plan / goal / default):
|
|
* the effective model and mode, and whether the mode came from an
|
|
explicit tier mode or from the global mode,
|
|
* tiers that were dropped, and why (invalid tier mode, no model,
|
|
unrecognised entry) -- a dropped tier is a BLOCKER, not a warning
|
|
to skim past, because applying the policy would silently not route
|
|
that tier,
|
|
* tiers absent from the document (worker default, not routed),
|
|
* unknown tier names in the document (ignored by the parser),
|
|
* whether any tier would actually activate on next dispatch.
|
|
|
|
What it does NOT do, honestly:
|
|
* It does NOT validate model names against a provider or relay
|
|
catalog. Task models are deliberately not whitelisted in this
|
|
codebase (the relay catalog changes), so a model typo here is
|
|
reported as "unverified, not whitelisted" only if the caller
|
|
passes a known-model set; without one, model names are reported
|
|
as-is and the caller must check them against the relay catalog
|
|
themselves. This module never invents a catalog.
|
|
* It does NOT check shadow sample counts. That is the shadow
|
|
report's job (GET /routing-shadow); a dry-run that said "ready
|
|
to activate" without sample data would be fabrication.
|
|
* It does NOT change host env, worker state, or any file. The CLI
|
|
exits 1 when the policy is invalid or any tier was dropped or
|
|
unknown, so a deploy script can gate on it; warnings alone
|
|
(no version, fallback_models present) do not block, but are
|
|
always printed.
|
|
|
|
Loading order, mirrored from load_routing_policy in models.py:
|
|
file (CLOUD_ROUTING_POLICY_FILE) first, then JSON
|
|
(CLOUD_ROUTING_POLICY_JSON) overrides whole tier entries (not
|
|
per-field: a JSON tier entry replaces the file's tier entry
|
|
entirely), then CLOUD_ROUTING_MODE overrides the GLOBAL mode
|
|
only -- it never overrides an explicit tier mode. Failed file/JSON
|
|
sources, which the loader silently ignores, are reported here as
|
|
warnings instead of being swallowed.
|
|
"""
|
|
|
|
import argparse
|
|
import json
|
|
import os
|
|
import sys
|
|
|
|
try:
|
|
from cloud.models import ROUTING_MODES, ROUTING_TIERS
|
|
from cloud.models import parse_routing_policy
|
|
except ImportError: # run as a script from the repo root
|
|
sys.path.insert(0, os.path.join(
|
|
os.path.dirname(os.path.abspath(__file__)), ".."))
|
|
from cloud.models import ROUTING_MODES, ROUTING_TIERS
|
|
from cloud.models import parse_routing_policy
|
|
|
|
|
|
def _tier_dropped_reason(raw_entry):
|
|
if isinstance(raw_entry, dict):
|
|
if "mode" in raw_entry and \
|
|
raw_entry.get("mode") not in ROUTING_MODES:
|
|
return "invalid tier mode %r" % (raw_entry.get("mode"),)
|
|
return "no usable model"
|
|
if isinstance(raw_entry, str):
|
|
return "empty model" if not raw_entry.strip() else "unknown"
|
|
return "unrecognised tier entry (expected string or dict)"
|
|
|
|
|
|
def dry_run_policy_document(data, env_mode_override=None):
|
|
"""Dry-run one merged policy document (pure, no IO).
|
|
|
|
``data`` is the merged document as parse_routing_policy would
|
|
receive it. ``env_mode_override`` is the raw CLOUD_ROUTING_MODE
|
|
value (or None): a valid value overrides the global mode only,
|
|
an invalid value is ignored with a warning.
|
|
"""
|
|
warnings = []
|
|
tiers_report = {}
|
|
dropped = []
|
|
unknown = []
|
|
if not isinstance(data, dict):
|
|
return {
|
|
"valid": False, "blocked": True, "version": None,
|
|
"global_mode": None, "global_mode_source": None,
|
|
"tiers": tiers_report, "dropped_tiers": dropped,
|
|
"unknown_tiers": unknown,
|
|
"warnings": ["policy document is not an object; "
|
|
"the loader would ignore it entirely"],
|
|
"would_activate": [], "would_shadow": [], "would_off": [],
|
|
}
|
|
raw_tiers = data.get("tiers")
|
|
if isinstance(raw_tiers, dict):
|
|
for name in sorted(raw_tiers):
|
|
if name not in ROUTING_TIERS:
|
|
unknown.append(name)
|
|
warnings.append(
|
|
"unknown tier %r is ignored by the parser" % name)
|
|
policy = parse_routing_policy(data)
|
|
if policy is None:
|
|
warnings.append(
|
|
"policy is unusable (no usable tiers, or global mode "
|
|
"outside off/shadow/active); the loader would ignore it "
|
|
"entirely and workers would keep the worker default")
|
|
return {
|
|
"valid": False, "blocked": True, "version": None,
|
|
"global_mode": None, "global_mode_source": None,
|
|
"tiers": tiers_report, "dropped_tiers": dropped,
|
|
"unknown_tiers": unknown, "warnings": warnings,
|
|
"would_activate": [], "would_shadow": [], "would_off": [],
|
|
}
|
|
global_mode = policy.mode
|
|
global_source = "policy"
|
|
if env_mode_override:
|
|
if env_mode_override in ROUTING_MODES:
|
|
global_mode = env_mode_override
|
|
global_source = "env_override"
|
|
warnings.append(
|
|
"CLOUD_ROUTING_MODE overrides the global mode to %r, "
|
|
"but does NOT override explicit tier modes" %
|
|
env_mode_override)
|
|
else:
|
|
warnings.append(
|
|
"CLOUD_ROUTING_MODE %r is invalid and ignored; "
|
|
"the global mode from the policy stands" %
|
|
env_mode_override)
|
|
# Apply the env global override to the effective per-tier modes
|
|
# without mutating the parsed policy: a tier with an explicit
|
|
# mode keeps it, a tier without one follows the overridden global.
|
|
effective_policy_mode = global_mode
|
|
for tier in ROUTING_TIERS:
|
|
raw_entry = raw_tiers.get(tier) if isinstance(raw_tiers,
|
|
dict) else None
|
|
if tier not in policy.tiers:
|
|
if raw_entry is not None:
|
|
reason = _tier_dropped_reason(raw_entry)
|
|
dropped.append(tier)
|
|
warnings.append(
|
|
"tier %r was dropped: %s; applying this policy "
|
|
"would silently not route %r tasks" %
|
|
(tier, reason, tier))
|
|
tiers_report[tier] = {
|
|
"model": None, "mode": None,
|
|
"mode_source": None, "status": "dropped",
|
|
"dropped_reason": reason,
|
|
}
|
|
else:
|
|
tiers_report[tier] = {
|
|
"model": None, "mode": None,
|
|
"mode_source": None, "status": "absent",
|
|
"dropped_reason": None,
|
|
}
|
|
continue
|
|
if tier in policy.tier_modes:
|
|
mode = policy.tier_modes[tier]
|
|
mode_source = "tier_mode"
|
|
else:
|
|
mode = effective_policy_mode
|
|
mode_source = "global_mode"
|
|
status = "active" if mode == "active" else (
|
|
"shadow" if mode == "shadow" else "off")
|
|
tiers_report[tier] = {
|
|
"model": policy.tiers[tier], "mode": mode,
|
|
"mode_source": mode_source, "status": status,
|
|
"dropped_reason": None,
|
|
}
|
|
would_activate = [t for t in ROUTING_TIERS
|
|
if tiers_report[t]["status"] == "active"]
|
|
would_shadow = [t for t in ROUTING_TIERS
|
|
if tiers_report[t]["status"] == "shadow"]
|
|
would_off = [t for t in ROUTING_TIERS
|
|
if tiers_report[t]["status"] == "off"]
|
|
if policy.version is None:
|
|
warnings.append(
|
|
"no policy version: policy changes will be hard to "
|
|
"attribute in shadow reports (decisions record "
|
|
"policy_version)")
|
|
if data.get("fallback_models") is not None:
|
|
warnings.append(
|
|
"fallback_models in the policy are parsed but NOT "
|
|
"applied; only worker CLI --fallback-models enables a "
|
|
"real fallback retry")
|
|
if would_activate:
|
|
warnings.append(
|
|
"activation warning: tier(s) %s would actually route "
|
|
"tasks on next worker dispatch if this policy were "
|
|
"applied; this dry-run does not activate anything" %
|
|
", ".join(would_activate))
|
|
blocked = bool(dropped or unknown)
|
|
return {
|
|
"valid": True, "blocked": blocked,
|
|
"version": policy.version, "global_mode": global_mode,
|
|
"global_mode_source": global_source,
|
|
"tiers": tiers_report, "dropped_tiers": dropped,
|
|
"unknown_tiers": unknown, "warnings": warnings,
|
|
"would_activate": would_activate,
|
|
"would_shadow": would_shadow, "would_off": would_off,
|
|
}
|
|
|
|
|
|
def dry_run_from_env(env=None):
|
|
"""Dry-run from an env mapping, mirroring load_routing_policy.
|
|
|
|
Returns the report dict. File/JSON failures that the loader
|
|
silently ignores are reported as warnings here.
|
|
"""
|
|
env = env if env is not None else os.environ
|
|
warnings = []
|
|
merged = None
|
|
path = env.get("CLOUD_ROUTING_POLICY_FILE") or ""
|
|
if path:
|
|
try:
|
|
with open(path, encoding="utf-8") as fh:
|
|
merged = json.load(fh)
|
|
except (OSError, ValueError) as exc:
|
|
warnings.append(
|
|
"policy file %r could not be read/parsed (%s); "
|
|
"the loader would ignore it" % (path, exc))
|
|
merged = None
|
|
raw = env.get("CLOUD_ROUTING_POLICY_JSON") or ""
|
|
if raw:
|
|
try:
|
|
override = json.loads(raw)
|
|
except ValueError as exc:
|
|
warnings.append(
|
|
"CLOUD_ROUTING_POLICY_JSON could not be parsed "
|
|
"(%s); the loader would ignore it" % exc)
|
|
override = None
|
|
if isinstance(override, dict):
|
|
if not isinstance(merged, dict):
|
|
merged = override
|
|
else:
|
|
merged = dict(merged)
|
|
merged.update(override)
|
|
if isinstance(merged.get("tiers"), dict) and \
|
|
isinstance(override.get("tiers"), dict):
|
|
tiers = dict(merged["tiers"])
|
|
tiers.update(override["tiers"])
|
|
merged["tiers"] = tiers
|
|
report = dry_run_policy_document(
|
|
merged, env_mode_override=env.get("CLOUD_ROUTING_MODE") or None)
|
|
# Source-failure warnings belong before the document warnings,
|
|
# and they do not change blocked (a failed source alone is not
|
|
# a dropped tier; the document report already covers validity).
|
|
report["warnings"] = warnings + report["warnings"]
|
|
return report
|
|
|
|
|
|
def main(argv=None):
|
|
parser = argparse.ArgumentParser(
|
|
description="Dry-run a routing policy (never applies it).")
|
|
parser.add_argument("--policy-file", default=None,
|
|
help="Policy JSON file (same as "
|
|
"CLOUD_ROUTING_POLICY_FILE)")
|
|
parser.add_argument("--policy-json", default=None,
|
|
help="Policy JSON string (same as "
|
|
"CLOUD_ROUTING_POLICY_JSON)")
|
|
parser.add_argument("--mode", default=None,
|
|
help="Global mode override (same as "
|
|
"CLOUD_ROUTING_MODE)")
|
|
args = parser.parse_args(argv)
|
|
env = {}
|
|
if args.policy_file:
|
|
env["CLOUD_ROUTING_POLICY_FILE"] = args.policy_file
|
|
if args.policy_json:
|
|
env["CLOUD_ROUTING_POLICY_JSON"] = args.policy_json
|
|
if args.mode:
|
|
env["CLOUD_ROUTING_MODE"] = args.mode
|
|
report = dry_run_from_env(env)
|
|
print(json.dumps(report, indent=2, sort_keys=True))
|
|
return 1 if (not report["valid"] or report["blocked"]) else 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|