feat: per-tier routing activation (v0c)
RoutingPolicy gains tier_modes: a tier entry may be a string
(model, mode inherits global) or {model, mode}. The explicit
tier mode overrides the global mode in both directions, so plan
can be active while goal/default stay shadow, and one tier can
be rolled back to off without touching the others. An invalid
tier mode drops that tier entirely (fail-closed: a typo must
never accidentally activate a tier, and never kills other
tiers). Tier-off tasks return worker_default and are not
audited as policy/shadow, so they never inflate shadow sample
counts. Explicit task and worker-CLI models still win over any
tier mode.
Tests: test_routing_tier.py (5 tests).
This commit is contained in:
1 parent
bac2a39377
commit
38357c472d
2 files changed
+171
-14
No files matched your search
+64
-14
@@ -178,21 +178,38 @@ def _clean_tiers(raw):
|
|||||||
class RoutingPolicy:
|
class RoutingPolicy:
|
||||||
"""A parsed, versioned routing policy (or an empty one).
|
"""A parsed, versioned routing policy (or an empty one).
|
||||||
|
|
||||||
``mode`` is one of ``off`` / ``shadow`` / ``active``. ``tiers`` maps
|
``mode`` is one of ``off`` / ``shadow`` / ``active`` and is the
|
||||||
a tier name to a model name; a tier absent from the map falls back
|
default for every tier. ``tiers`` maps a tier name to a model
|
||||||
to the worker default. ``fallback_models`` is parsed and validated
|
name; a tier absent from the map falls back to the worker
|
||||||
but deliberately NOT applied by v0: the worker's explicit
|
default. ``tier_modes`` (v0c, per-tier activation) maps a tier
|
||||||
``--fallback-models`` remains the only source that can enable a real
|
name to an explicit mode that overrides the global ``mode`` in
|
||||||
fallback retry, so a policy file alone can never start burning a
|
both directions: a tier can be ``active`` while the global mode
|
||||||
second attempt's quota.
|
is ``shadow``/``off`` (activate plan first), and a tier can be
|
||||||
|
``off`` while the global mode is ``active`` (roll back one tier
|
||||||
|
without touching the others). A tier entry whose mode is not a
|
||||||
|
valid mode is dropped entirely by the parser (fail-closed: a
|
||||||
|
typo must never accidentally activate a tier).
|
||||||
|
``fallback_models`` is parsed and validated but deliberately
|
||||||
|
NOT applied by v0: the worker's explicit ``--fallback-models``
|
||||||
|
remains the only source that can enable a real fallback retry,
|
||||||
|
so a policy file alone can never start burning a second
|
||||||
|
attempt's quota.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self, version=None, mode="shadow", tiers=None,
|
def __init__(self, version=None, mode="shadow", tiers=None,
|
||||||
fallback_models=None):
|
fallback_models=None, tier_modes=None):
|
||||||
self.version = version if isinstance(version, str) \
|
self.version = version if isinstance(version, str) \
|
||||||
and version.strip() else None
|
and version.strip() else None
|
||||||
self.mode = mode if mode in ROUTING_MODES else "off"
|
self.mode = mode if mode in ROUTING_MODES else "off"
|
||||||
self.tiers = dict(tiers or {})
|
self.tiers = dict(tiers or {})
|
||||||
|
self.tier_modes = {}
|
||||||
|
for tier, tier_mode in (tier_modes or {}).items():
|
||||||
|
if tier in self.tiers and tier_mode in ROUTING_MODES:
|
||||||
|
self.tier_modes[tier] = tier_mode
|
||||||
|
|
||||||
|
def tier_mode(self, tier):
|
||||||
|
"""Effective mode for one tier: explicit tier mode, else global."""
|
||||||
|
return self.tier_modes.get(tier, self.mode)
|
||||||
self.fallback_models = []
|
self.fallback_models = []
|
||||||
for item in fallback_models or []:
|
for item in fallback_models or []:
|
||||||
model = _clean_model_name(item)
|
model = _clean_model_name(item)
|
||||||
@@ -215,6 +232,24 @@ def parse_routing_policy(data):
|
|||||||
if mode not in ROUTING_MODES:
|
if mode not in ROUTING_MODES:
|
||||||
return None
|
return None
|
||||||
tiers = _clean_tiers(data.get("tiers"))
|
tiers = _clean_tiers(data.get("tiers"))
|
||||||
|
if not tiers:
|
||||||
|
return None
|
||||||
|
# Per-tier modes (v0c): a tier entry may be a string (model,
|
||||||
|
# mode inherits global) or a dict {"model": ..., "mode": ...}.
|
||||||
|
# An invalid tier mode drops the whole tier entry (fail-closed),
|
||||||
|
# it never falls back to the global mode where a typo could
|
||||||
|
# accidentally activate, and never kills the other tiers.
|
||||||
|
raw_tiers = data.get("tiers") if isinstance(data.get("tiers"),
|
||||||
|
dict) else {}
|
||||||
|
tier_modes = {}
|
||||||
|
for tier, entry in raw_tiers.items():
|
||||||
|
if not isinstance(entry, dict) or "mode" not in entry:
|
||||||
|
continue
|
||||||
|
tier_mode = entry.get("mode")
|
||||||
|
if tier_mode in ROUTING_MODES:
|
||||||
|
tier_modes[tier] = tier_mode
|
||||||
|
else:
|
||||||
|
tiers.pop(tier, None)
|
||||||
if not tiers:
|
if not tiers:
|
||||||
return None
|
return None
|
||||||
fallback_models = data.get("fallback_models")
|
fallback_models = data.get("fallback_models")
|
||||||
@@ -222,7 +257,8 @@ def parse_routing_policy(data):
|
|||||||
list):
|
list):
|
||||||
return None
|
return None
|
||||||
return RoutingPolicy(version=data.get("version"), mode=mode,
|
return RoutingPolicy(version=data.get("version"), mode=mode,
|
||||||
tiers=tiers, fallback_models=fallback_models)
|
tiers=tiers, fallback_models=fallback_models,
|
||||||
|
tier_modes=tier_modes)
|
||||||
|
|
||||||
|
|
||||||
def load_routing_policy(env=None):
|
def load_routing_policy(env=None):
|
||||||
@@ -305,7 +341,10 @@ def decide_routing(task, worker_model, policy=None,
|
|||||||
1. task-level ``model`` (explicit) -- never overridden;
|
1. task-level ``model`` (explicit) -- never overridden;
|
||||||
2. worker CLI ``--model`` (explicit) -- a pinned deployment is never
|
2. worker CLI ``--model`` (explicit) -- a pinned deployment is never
|
||||||
silently changed by a policy file;
|
silently changed by a policy file;
|
||||||
3. active policy tier model -- only when the task named no model;
|
3. policy tier model in the tier's effective mode (v0c: explicit
|
||||||
|
tier mode, else the global mode) -- only when the task named
|
||||||
|
no model; ``active`` applies the tier model, ``shadow``
|
||||||
|
suggests it, ``off`` falls back to the worker default;
|
||||||
4. worker default chain (config file / ``DEFAULT_MODEL``).
|
4. worker default chain (config file / ``DEFAULT_MODEL``).
|
||||||
|
|
||||||
Shadow mode returns the worker default as ``chosen_model`` and the
|
Shadow mode returns the worker default as ``chosen_model`` and the
|
||||||
@@ -335,11 +374,11 @@ def decide_routing(task, worker_model, policy=None,
|
|||||||
suggested_model=None, source="worker_explicit",
|
suggested_model=None, source="worker_explicit",
|
||||||
policy_version=None, applied=True,
|
policy_version=None, applied=True,
|
||||||
reason="worker CLI model explicit", signals=signals)
|
reason="worker CLI model explicit", signals=signals)
|
||||||
if policy is None or policy.mode == "off":
|
if policy is None:
|
||||||
return RoutingDecision(
|
return RoutingDecision(
|
||||||
task_id=task.get("id"), tier=tier, chosen_model=worker_model,
|
task_id=task.get("id"), tier=tier, chosen_model=worker_model,
|
||||||
suggested_model=None, source="worker_default",
|
suggested_model=None, source="worker_default",
|
||||||
policy_version=getattr(policy, "version", None),
|
policy_version=None,
|
||||||
applied=True, reason="routing off or no policy",
|
applied=True, reason="routing off or no policy",
|
||||||
signals=signals)
|
signals=signals)
|
||||||
tier_model = policy.tiers.get(tier)
|
tier_model = policy.tiers.get(tier)
|
||||||
@@ -349,14 +388,25 @@ def decide_routing(task, worker_model, policy=None,
|
|||||||
suggested_model=None, source="worker_default",
|
suggested_model=None, source="worker_default",
|
||||||
policy_version=policy.version, applied=True,
|
policy_version=policy.version, applied=True,
|
||||||
reason="%s tier has no policy model" % tier, signals=signals)
|
reason="%s tier has no policy model" % tier, signals=signals)
|
||||||
if policy.mode == "shadow":
|
# Per-tier effective mode (v0c): explicit tier mode overrides
|
||||||
|
# the global mode in both directions. A tier in "off" is not
|
||||||
|
# routed at all (no policy/shadow audit downstream, so it never
|
||||||
|
# inflates a shadow sample count).
|
||||||
|
effective_mode = policy.tier_mode(tier)
|
||||||
|
if effective_mode == "off":
|
||||||
|
return RoutingDecision(
|
||||||
|
task_id=task.get("id"), tier=tier, chosen_model=worker_model,
|
||||||
|
suggested_model=None, source="worker_default",
|
||||||
|
policy_version=policy.version, applied=True,
|
||||||
|
reason="%s tier mode is off" % tier, signals=signals)
|
||||||
|
if effective_mode == "shadow":
|
||||||
return RoutingDecision(
|
return RoutingDecision(
|
||||||
task_id=task.get("id"), tier=tier, chosen_model=worker_model,
|
task_id=task.get("id"), tier=tier, chosen_model=worker_model,
|
||||||
suggested_model=tier_model, source="shadow",
|
suggested_model=tier_model, source="shadow",
|
||||||
policy_version=policy.version, applied=False,
|
policy_version=policy.version, applied=False,
|
||||||
reason="%s tier suggests %s (shadow, not applied)"
|
reason="%s tier suggests %s (shadow, not applied)"
|
||||||
% (tier, tier_model), signals=signals)
|
% (tier, tier_model), signals=signals)
|
||||||
# mode == "active"
|
# effective_mode == "active"
|
||||||
return RoutingDecision(
|
return RoutingDecision(
|
||||||
task_id=task.get("id"), tier=tier, chosen_model=tier_model,
|
task_id=task.get("id"), tier=tier, chosen_model=tier_model,
|
||||||
suggested_model=None, source="policy",
|
suggested_model=None, source="policy",
|
||||||
|
|||||||
@@ -0,0 +1,107 @@
|
|||||||
|
"""分档激活 v0c 单测:档模式优先全局、字符串兼容、无效档模式丢档。
|
||||||
|
|
||||||
|
运行(在仓库根目录下):
|
||||||
|
python3 -m unittest discover -s cloud/tests -p "test_routing_tier.py" -v
|
||||||
|
|
||||||
|
说明:纯函数单测,不碰库、不联网、不跑模型。关键断言:
|
||||||
|
全局 shadow 时 plan 档 active 只切 plan 档、其余档仍影子;
|
||||||
|
全局 active 时某档 off 只回滚该档;档模式写错时该档整档
|
||||||
|
丢掉、不回落全局(防 typo 误激活),其余档不受影响;
|
||||||
|
字符串档写法继续按全局模式走。
|
||||||
|
"""
|
||||||
|
|
||||||
|
import sys
|
||||||
|
import unittest
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
try:
|
||||||
|
from cloud.models import decide_routing, parse_routing_policy
|
||||||
|
except ImportError:
|
||||||
|
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||||||
|
from cloud.models import decide_routing, parse_routing_policy
|
||||||
|
|
||||||
|
|
||||||
|
def _policy(data):
|
||||||
|
policy = parse_routing_policy(data)
|
||||||
|
assert policy is not None, data
|
||||||
|
return policy
|
||||||
|
|
||||||
|
|
||||||
|
class TierModeTestCase(unittest.TestCase):
|
||||||
|
def test_plan_active_only_under_global_shadow(self):
|
||||||
|
policy = _policy({
|
||||||
|
"mode": "shadow",
|
||||||
|
"tiers": {
|
||||||
|
"plan": {"model": "gpt-6.1-sol", "mode": "active"},
|
||||||
|
"goal": "gpt-6.1-sol",
|
||||||
|
"default": "gpt-6.1-sol",
|
||||||
|
},
|
||||||
|
})
|
||||||
|
plan = decide_routing({"id": "t1", "plan": True}, "gpt-5.6-terra",
|
||||||
|
policy=policy)
|
||||||
|
self.assertEqual(plan.source, "policy")
|
||||||
|
self.assertTrue(plan.applied)
|
||||||
|
self.assertEqual(plan.chosen_model, "gpt-6.1-sol")
|
||||||
|
goal = decide_routing({"id": "t2", "goal_mode": True},
|
||||||
|
"gpt-5.6-terra", policy=policy)
|
||||||
|
self.assertEqual(goal.source, "shadow")
|
||||||
|
self.assertFalse(goal.applied)
|
||||||
|
self.assertEqual(goal.suggested_model, "gpt-6.1-sol")
|
||||||
|
self.assertEqual(goal.chosen_model, "gpt-5.6-terra")
|
||||||
|
|
||||||
|
def test_tier_off_rolls_back_only_that_tier(self):
|
||||||
|
policy = _policy({
|
||||||
|
"mode": "active",
|
||||||
|
"tiers": {
|
||||||
|
"plan": {"model": "gpt-6.1-sol", "mode": "off"},
|
||||||
|
"default": "gpt-6.1-sol",
|
||||||
|
},
|
||||||
|
})
|
||||||
|
plan = decide_routing({"id": "t1", "plan": True}, "gpt-5.6-terra",
|
||||||
|
policy=policy)
|
||||||
|
self.assertEqual(plan.source, "worker_default")
|
||||||
|
self.assertEqual(plan.chosen_model, "gpt-5.6-terra")
|
||||||
|
default = decide_routing({"id": "t2"}, "gpt-5.6-terra",
|
||||||
|
policy=policy)
|
||||||
|
self.assertEqual(default.source, "policy")
|
||||||
|
self.assertEqual(default.chosen_model, "gpt-6.1-sol")
|
||||||
|
|
||||||
|
def test_string_tier_inherits_global_mode(self):
|
||||||
|
policy = _policy({
|
||||||
|
"mode": "shadow",
|
||||||
|
"tiers": {"plan": "gpt-6.1-sol"},
|
||||||
|
})
|
||||||
|
decision = decide_routing({"id": "t1", "plan": True},
|
||||||
|
"gpt-5.6-terra", policy=policy)
|
||||||
|
self.assertEqual(decision.source, "shadow")
|
||||||
|
self.assertFalse(decision.applied)
|
||||||
|
|
||||||
|
def test_invalid_tier_mode_drops_tier_not_global(self):
|
||||||
|
policy = _policy({
|
||||||
|
"mode": "active",
|
||||||
|
"tiers": {
|
||||||
|
"plan": {"model": "gpt-6.1-sol", "mode": "activ"},
|
||||||
|
"default": "gpt-6.1-sol",
|
||||||
|
},
|
||||||
|
})
|
||||||
|
self.assertNotIn("plan", policy.tiers)
|
||||||
|
plan = decide_routing({"id": "t1", "plan": True}, "gpt-5.6-terra",
|
||||||
|
policy=policy)
|
||||||
|
# 丢档 = 回 worker 默认,不按全局 active 误激活。
|
||||||
|
self.assertEqual(plan.source, "worker_default")
|
||||||
|
self.assertEqual(plan.chosen_model, "gpt-5.6-terra")
|
||||||
|
default = decide_routing({"id": "t2"}, "gpt-5.6-terra",
|
||||||
|
policy=policy)
|
||||||
|
self.assertEqual(default.source, "policy")
|
||||||
|
|
||||||
|
def test_explicit_task_and_worker_still_win(self):
|
||||||
|
policy = _policy({
|
||||||
|
"mode": "active",
|
||||||
|
"tiers": {"plan": {"model": "gpt-6.1-sol",
|
||||||
|
"mode": "active"}},
|
||||||
|
})
|
||||||
|
explicit = decide_routing(
|
||||||
|
{"id": "t1", "plan": True, "model": "gpt-5.6-terra"},
|
||||||
|
"gpt-5.6-terra", policy=policy)
|
||||||
|
self.assertEqual(explicit.source, "explicit")
|
||||||
|
self.assertEqual(explicit.chosen_model, "gpt-5.6-terra")
|
||||||
Reference in new issue
Block a user