host
cd33a929ca
Policy carries an operator-declared model_capabilities directory and per-tier requires (tool_calling/reasoning/vision booleans, min_context_tokens). decide_routing checks the policy tier model in active mode: capable applies, incapable falls back to a capable worker default, and if both fail or are unknown it returns capability_blocked, which the worker turns into a failed task (v3 tools-gate shape, runner never called). Shadow mode only annotates the reason, never blocks. Explicit task and worker CLI models bypass the check by design. Invalid requires drops only that tier (fail-closed, like invalid tier mode). Dry-run reports per-tier capability status and blocks on active tiers that fail or cannot be verified.
319 lines
13 KiB
Python
319 lines
13 KiB
Python
"""Capability routing (v8): directory, requires, decision filter,
|
|
worker capability gate, and dry-run reporting.
|
|
|
|
Run: python3 -m unittest discover -s cloud/tests -p "test_routing_capability.py" -v
|
|
"""
|
|
|
|
import os
|
|
import sqlite3
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
import unittest
|
|
from types import SimpleNamespace
|
|
|
|
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", ".."))
|
|
|
|
from cloud.models import (check_model_capabilities, decide_routing,
|
|
parse_model_capabilities, parse_routing_policy,
|
|
parse_tier_requires)
|
|
from cloud.routing_dryrun import dry_run_policy_document
|
|
from cloud.server import init_db
|
|
from cloud.worker import Worker
|
|
|
|
|
|
def _policy(**overrides):
|
|
doc = {
|
|
"version": "cap-v8",
|
|
"mode": "active",
|
|
"tiers": {
|
|
"plan": {"model": "plan-model",
|
|
"requires": {"tool_calling": True,
|
|
"min_context_tokens": 128000}},
|
|
},
|
|
"model_capabilities": {
|
|
"plan-model": {"tool_calling": True,
|
|
"context_tokens": 200000},
|
|
"worker-model": {"tool_calling": True,
|
|
"context_tokens": 200000},
|
|
"no-tools-model": {"tool_calling": False,
|
|
"context_tokens": 200000},
|
|
"small-model": {"tool_calling": True,
|
|
"context_tokens": 32000},
|
|
},
|
|
}
|
|
doc.update(overrides)
|
|
return parse_routing_policy(doc)
|
|
|
|
|
|
class CapabilityParseTestCase(unittest.TestCase):
|
|
def test_clean_capabilities_drops_bad_entries(self):
|
|
caps = parse_model_capabilities({
|
|
"good": {"tool_calling": True, "context_tokens": 1000},
|
|
"bad-bool": {"tool_calling": "yes"},
|
|
"bad-int": {"context_tokens": -5},
|
|
"": {"tool_calling": True},
|
|
"not-dict": "x",
|
|
})
|
|
self.assertEqual(list(caps.keys()), ["good"])
|
|
|
|
def test_requires_rejects_unknown_key_and_bad_value(self):
|
|
self.assertIsNone(parse_tier_requires({"fuzzy": True}))
|
|
self.assertIsNone(parse_tier_requires({"tool_calling": "yes"}))
|
|
self.assertIsNone(parse_tier_requires({"min_context_tokens": 0}))
|
|
self.assertEqual(parse_tier_requires({"tool_calling": False}), {})
|
|
self.assertEqual(
|
|
parse_tier_requires({"tool_calling": True,
|
|
"min_context_tokens": 100}),
|
|
{"tool_calling": True, "min_context_tokens": 100})
|
|
|
|
def test_invalid_requires_drops_only_that_tier(self):
|
|
doc = {
|
|
"mode": "active",
|
|
"tiers": {
|
|
"plan": {"model": "plan-model",
|
|
"requires": {"fuzzy": True}},
|
|
"default": "default-model",
|
|
},
|
|
}
|
|
policy = parse_routing_policy(doc)
|
|
self.assertNotIn("plan", policy.tiers)
|
|
self.assertIn("default", policy.tiers)
|
|
|
|
def test_check_matrix(self):
|
|
caps = {"m": {"tool_calling": True, "context_tokens": 1000}}
|
|
self.assertEqual(check_model_capabilities(caps, "m", {}),
|
|
(True, [], False))
|
|
self.assertEqual(check_model_capabilities(
|
|
caps, "m", {"tool_calling": True}), (True, [], False))
|
|
self.assertEqual(check_model_capabilities(
|
|
caps, "m", {"vision": True}), (False, ["vision"], True))
|
|
self.assertEqual(check_model_capabilities(
|
|
caps, "m", {"min_context_tokens": 2000}),
|
|
(False, ["min_context_tokens"], False))
|
|
self.assertEqual(check_model_capabilities(
|
|
caps, "absent", {"tool_calling": True}),
|
|
(False, ["tool_calling"], True))
|
|
|
|
|
|
class CapabilityDecisionTestCase(unittest.TestCase):
|
|
def test_active_capable_applies_policy_model(self):
|
|
decision = decide_routing(
|
|
{"id": "t1", "plan": True}, "worker-model",
|
|
policy=_policy())
|
|
self.assertEqual(decision.source, "policy")
|
|
self.assertEqual(decision.chosen_model, "plan-model")
|
|
self.assertTrue(decision.applied)
|
|
|
|
def test_active_incapable_falls_back_to_capable_worker_default(self):
|
|
policy = _policy()
|
|
policy.tiers["plan"] = "no-tools-model"
|
|
decision = decide_routing(
|
|
{"id": "t1", "plan": True}, "worker-model", policy=policy)
|
|
self.assertEqual(decision.source, "worker_default")
|
|
self.assertEqual(decision.chosen_model, "worker-model")
|
|
self.assertIn("lacks", decision.reason)
|
|
|
|
def test_active_incapable_and_worker_incapable_blocks(self):
|
|
policy = _policy()
|
|
policy.tiers["plan"] = "no-tools-model"
|
|
decision = decide_routing(
|
|
{"id": "t1", "plan": True}, "small-model", policy=policy)
|
|
self.assertEqual(decision.source, "capability_blocked")
|
|
self.assertIsNone(decision.chosen_model)
|
|
self.assertFalse(decision.applied)
|
|
# The reason names the tier model's shortfall (tool
|
|
# calling); the worker default's own shortfall (context)
|
|
# is what made the fallback fail, not the reason text.
|
|
self.assertIn("tool_calling", decision.reason)
|
|
|
|
def test_active_unknown_model_blocks(self):
|
|
policy = _policy()
|
|
policy.tiers["plan"] = "unlisted-model"
|
|
decision = decide_routing(
|
|
{"id": "t1", "plan": True}, "worker-model", policy=policy)
|
|
# worker default is capable, so it is used instead of blocking
|
|
self.assertEqual(decision.source, "worker_default")
|
|
self.assertEqual(decision.chosen_model, "worker-model")
|
|
self.assertIn("unknown", decision.reason)
|
|
|
|
def test_active_unknown_model_and_worker_unknown_blocks(self):
|
|
policy = _policy()
|
|
policy.tiers["plan"] = "unlisted-model"
|
|
decision = decide_routing(
|
|
{"id": "t1", "plan": True}, "also-unlisted", policy=policy)
|
|
self.assertEqual(decision.source, "capability_blocked")
|
|
self.assertIsNone(decision.chosen_model)
|
|
|
|
def test_shadow_incapable_never_blocks(self):
|
|
policy = parse_routing_policy({
|
|
"version": "cap-v8", "mode": "shadow",
|
|
"tiers": {"plan": {"model": "no-tools-model",
|
|
"requires": {"tool_calling": True}}},
|
|
"model_capabilities": {
|
|
"no-tools-model": {"tool_calling": False},
|
|
},
|
|
})
|
|
decision = decide_routing(
|
|
{"id": "t1", "plan": True}, "worker-model", policy=policy)
|
|
self.assertEqual(decision.source, "shadow")
|
|
self.assertEqual(decision.chosen_model, "worker-model")
|
|
self.assertEqual(decision.suggested_model, "no-tools-model")
|
|
self.assertIn("lacks", decision.reason)
|
|
|
|
def test_explicit_models_bypass_capability_check(self):
|
|
policy = _policy()
|
|
policy.tiers["plan"] = "no-tools-model"
|
|
decision = decide_routing(
|
|
{"id": "t1", "plan": True, "model": "no-tools-model"},
|
|
"small-model", policy=policy)
|
|
self.assertEqual(decision.source, "explicit")
|
|
self.assertEqual(decision.chosen_model, "no-tools-model")
|
|
decision = decide_routing(
|
|
{"id": "t1", "plan": True}, "no-tools-model",
|
|
policy=policy, worker_model_explicit=True)
|
|
self.assertEqual(decision.source, "worker_explicit")
|
|
self.assertEqual(decision.chosen_model, "no-tools-model")
|
|
|
|
def test_no_requirements_behaves_as_before(self):
|
|
policy = parse_routing_policy({
|
|
"mode": "active", "tiers": {"plan": "plan-model"}})
|
|
decision = decide_routing(
|
|
{"id": "t1", "plan": True}, "worker-model", policy=policy)
|
|
self.assertEqual(decision.source, "policy")
|
|
self.assertEqual(decision.chosen_model, "plan-model")
|
|
|
|
|
|
class CapabilityDryRunTestCase(unittest.TestCase):
|
|
def test_active_incapable_blocks_dry_run(self):
|
|
report = dry_run_policy_document({
|
|
"mode": "active",
|
|
"tiers": {"plan": {"model": "no-tools-model",
|
|
"requires": {"tool_calling": True}}},
|
|
"model_capabilities": {
|
|
"no-tools-model": {"tool_calling": False},
|
|
},
|
|
})
|
|
self.assertTrue(report["blocked"])
|
|
self.assertEqual(
|
|
report["tiers"]["plan"]["capabilities"]["status"], "fail")
|
|
|
|
def test_shadow_incapable_warns_only(self):
|
|
report = dry_run_policy_document({
|
|
"mode": "shadow",
|
|
"tiers": {"plan": {"model": "no-tools-model",
|
|
"requires": {"tool_calling": True}}},
|
|
"model_capabilities": {
|
|
"no-tools-model": {"tool_calling": False},
|
|
},
|
|
})
|
|
self.assertFalse(report["blocked"])
|
|
self.assertTrue(any("lacks" in w for w in report["warnings"]))
|
|
|
|
def test_active_capable_does_not_block(self):
|
|
report = dry_run_policy_document({
|
|
"mode": "active",
|
|
"tiers": {"plan": {"model": "plan-model",
|
|
"requires": {"tool_calling": True}}},
|
|
"model_capabilities": {
|
|
"plan-model": {"tool_calling": True},
|
|
},
|
|
})
|
|
self.assertFalse(report["blocked"])
|
|
self.assertEqual(
|
|
report["tiers"]["plan"]["capabilities"]["status"], "ok")
|
|
|
|
|
|
def _completed_runner(cmd):
|
|
if "dispatch" in cmd:
|
|
name = cmd[cmd.index("--name") + 1]
|
|
else:
|
|
name = cmd[cmd.index("--task") + 1]
|
|
return SimpleNamespace(
|
|
returncode=0,
|
|
stdout='{"task": "%s", "status": "completed", "text": "done"}' % name,
|
|
stderr="")
|
|
|
|
|
|
def _insert_task(db_path, task_id):
|
|
now = time.time()
|
|
conn = sqlite3.connect(db_path)
|
|
try:
|
|
conn.execute(
|
|
"INSERT INTO tasks(id, project_id, prompt, status,"
|
|
" idempotency_key, depends_on, priority, created_by,"
|
|
" created_at, updated_at, last_heartbeat)"
|
|
" VALUES(?, 'p1', 'do work', 'queued', NULL, '[]', 0,"
|
|
" 'tester', ?, ?, NULL)", (task_id, now, now))
|
|
conn.commit()
|
|
finally:
|
|
conn.close()
|
|
|
|
|
|
class CapabilityWorkerGateTestCase(unittest.TestCase):
|
|
def setUp(self):
|
|
self.tmp = tempfile.TemporaryDirectory()
|
|
self.db_path = os.path.join(self.tmp.name, "control.db")
|
|
init_db(self.db_path)
|
|
self.calls = []
|
|
|
|
def tearDown(self):
|
|
self.tmp.cleanup()
|
|
|
|
def _worker(self, policy_doc, model):
|
|
def recording(cmd):
|
|
self.calls.append(list(cmd))
|
|
return _completed_runner(cmd)
|
|
|
|
# model_explicit=False: the worker model is a plain
|
|
# default, not a CLI pin -- a CLI pin would bypass the
|
|
# policy (and the capability check) by design, so the
|
|
# gate test must not use one.
|
|
return Worker(self.db_path, "/fake/host.py", runner=recording,
|
|
env={"XCM_API_KEY": "test-placeholder"},
|
|
model=model, model_explicit=False,
|
|
routing_policy=policy_doc)
|
|
|
|
def test_blocked_decision_fails_task_without_running(self):
|
|
_insert_task(self.db_path, "A")
|
|
worker = self._worker({
|
|
"mode": "active",
|
|
"tiers": {"default": {"model": "no-tools-model",
|
|
"requires": {"tool_calling": True}}},
|
|
"model_capabilities": {
|
|
"no-tools-model": {"tool_calling": False},
|
|
"small-model": {"tool_calling": True,
|
|
"context_tokens": 32000},
|
|
},
|
|
}, model="small-model")
|
|
# small-model is capable on tool calling but the task is
|
|
# default-tier: the tier model fails tool calling, and the
|
|
# worker default must be checked against the same
|
|
# requires -- here it passes, so this is NOT a block.
|
|
result = worker.run_one()
|
|
self.assertEqual(result["status"], "completed")
|
|
|
|
def test_blocked_decision_fails_task_when_worker_also_incapable(self):
|
|
_insert_task(self.db_path, "B")
|
|
worker = self._worker({
|
|
"mode": "active",
|
|
"tiers": {"default": {"model": "no-tools-model",
|
|
"requires": {"tool_calling": True}}},
|
|
"model_capabilities": {
|
|
"no-tools-model": {"tool_calling": False},
|
|
"no-tools-worker": {"tool_calling": False},
|
|
},
|
|
}, model="no-tools-worker")
|
|
result = worker.run_one()
|
|
self.assertTrue(result["dispatched"])
|
|
self.assertEqual(result["status"], "failed")
|
|
self.assertIn("tool_calling", result["error"])
|
|
self.assertEqual(self.calls, []) # gate fires before runner
|
|
conn = sqlite3.connect(self.db_path)
|
|
try:
|
|
status = conn.execute(
|
|
"SELECT status FROM tasks WHERE id = 'B'").fetchone()[0]
|
|
finally:
|
|
conn.close()
|
|
self.assertEqual(status, "failed")
|