"""Capability routing (v8): directory, requires, decision filter, worker capability gate, and dry-run reporting. Run: python3 -m unittest discover -s cloud/tests -p "test_routing_capability.py" -v """ import os import sqlite3 import sys import tempfile import time import unittest from types import SimpleNamespace sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..")) from cloud.models import (check_model_capabilities, decide_routing, parse_model_capabilities, parse_routing_policy, parse_tier_requires) from cloud.routing_dryrun import dry_run_policy_document from cloud.server import init_db from cloud.worker import Worker def _policy(**overrides): doc = { "version": "cap-v8", "mode": "active", "tiers": { "plan": {"model": "plan-model", "requires": {"tool_calling": True, "min_context_tokens": 128000}}, }, "model_capabilities": { "plan-model": {"tool_calling": True, "context_tokens": 200000}, "worker-model": {"tool_calling": True, "context_tokens": 200000}, "no-tools-model": {"tool_calling": False, "context_tokens": 200000}, "small-model": {"tool_calling": True, "context_tokens": 32000}, }, } doc.update(overrides) return parse_routing_policy(doc) class CapabilityParseTestCase(unittest.TestCase): def test_clean_capabilities_drops_bad_entries(self): caps = parse_model_capabilities({ "good": {"tool_calling": True, "context_tokens": 1000}, "bad-bool": {"tool_calling": "yes"}, "bad-int": {"context_tokens": -5}, "": {"tool_calling": True}, "not-dict": "x", }) self.assertEqual(list(caps.keys()), ["good"]) def test_requires_rejects_unknown_key_and_bad_value(self): self.assertIsNone(parse_tier_requires({"fuzzy": True})) self.assertIsNone(parse_tier_requires({"tool_calling": "yes"})) self.assertIsNone(parse_tier_requires({"min_context_tokens": 0})) self.assertEqual(parse_tier_requires({"tool_calling": False}), {}) self.assertEqual( parse_tier_requires({"tool_calling": True, "min_context_tokens": 100}), {"tool_calling": True, "min_context_tokens": 100}) def test_invalid_requires_drops_only_that_tier(self): doc = { "mode": "active", "tiers": { "plan": {"model": "plan-model", "requires": {"fuzzy": True}}, "default": "default-model", }, } policy = parse_routing_policy(doc) self.assertNotIn("plan", policy.tiers) self.assertIn("default", policy.tiers) def test_check_matrix(self): caps = {"m": {"tool_calling": True, "context_tokens": 1000}} self.assertEqual(check_model_capabilities(caps, "m", {}), (True, [], False)) self.assertEqual(check_model_capabilities( caps, "m", {"tool_calling": True}), (True, [], False)) self.assertEqual(check_model_capabilities( caps, "m", {"vision": True}), (False, ["vision"], True)) self.assertEqual(check_model_capabilities( caps, "m", {"min_context_tokens": 2000}), (False, ["min_context_tokens"], False)) self.assertEqual(check_model_capabilities( caps, "absent", {"tool_calling": True}), (False, ["tool_calling"], True)) class CapabilityDecisionTestCase(unittest.TestCase): def test_active_capable_applies_policy_model(self): decision = decide_routing( {"id": "t1", "plan": True}, "worker-model", policy=_policy()) self.assertEqual(decision.source, "policy") self.assertEqual(decision.chosen_model, "plan-model") self.assertTrue(decision.applied) def test_active_incapable_falls_back_to_capable_worker_default(self): policy = _policy() policy.tiers["plan"] = "no-tools-model" decision = decide_routing( {"id": "t1", "plan": True}, "worker-model", policy=policy) self.assertEqual(decision.source, "worker_default") self.assertEqual(decision.chosen_model, "worker-model") self.assertIn("lacks", decision.reason) def test_active_incapable_and_worker_incapable_blocks(self): policy = _policy() policy.tiers["plan"] = "no-tools-model" decision = decide_routing( {"id": "t1", "plan": True}, "small-model", policy=policy) self.assertEqual(decision.source, "capability_blocked") self.assertIsNone(decision.chosen_model) self.assertFalse(decision.applied) # The reason names the tier model's shortfall (tool # calling); the worker default's own shortfall (context) # is what made the fallback fail, not the reason text. self.assertIn("tool_calling", decision.reason) def test_active_unknown_model_blocks(self): policy = _policy() policy.tiers["plan"] = "unlisted-model" decision = decide_routing( {"id": "t1", "plan": True}, "worker-model", policy=policy) # worker default is capable, so it is used instead of blocking self.assertEqual(decision.source, "worker_default") self.assertEqual(decision.chosen_model, "worker-model") self.assertIn("unknown", decision.reason) def test_active_unknown_model_and_worker_unknown_blocks(self): policy = _policy() policy.tiers["plan"] = "unlisted-model" decision = decide_routing( {"id": "t1", "plan": True}, "also-unlisted", policy=policy) self.assertEqual(decision.source, "capability_blocked") self.assertIsNone(decision.chosen_model) def test_shadow_incapable_never_blocks(self): policy = parse_routing_policy({ "version": "cap-v8", "mode": "shadow", "tiers": {"plan": {"model": "no-tools-model", "requires": {"tool_calling": True}}}, "model_capabilities": { "no-tools-model": {"tool_calling": False}, }, }) decision = decide_routing( {"id": "t1", "plan": True}, "worker-model", policy=policy) self.assertEqual(decision.source, "shadow") self.assertEqual(decision.chosen_model, "worker-model") self.assertEqual(decision.suggested_model, "no-tools-model") self.assertIn("lacks", decision.reason) def test_explicit_models_bypass_capability_check(self): policy = _policy() policy.tiers["plan"] = "no-tools-model" decision = decide_routing( {"id": "t1", "plan": True, "model": "no-tools-model"}, "small-model", policy=policy) self.assertEqual(decision.source, "explicit") self.assertEqual(decision.chosen_model, "no-tools-model") decision = decide_routing( {"id": "t1", "plan": True}, "no-tools-model", policy=policy, worker_model_explicit=True) self.assertEqual(decision.source, "worker_explicit") self.assertEqual(decision.chosen_model, "no-tools-model") def test_no_requirements_behaves_as_before(self): policy = parse_routing_policy({ "mode": "active", "tiers": {"plan": "plan-model"}}) decision = decide_routing( {"id": "t1", "plan": True}, "worker-model", policy=policy) self.assertEqual(decision.source, "policy") self.assertEqual(decision.chosen_model, "plan-model") class CapabilityDryRunTestCase(unittest.TestCase): def test_active_incapable_blocks_dry_run(self): report = dry_run_policy_document({ "mode": "active", "tiers": {"plan": {"model": "no-tools-model", "requires": {"tool_calling": True}}}, "model_capabilities": { "no-tools-model": {"tool_calling": False}, }, }) self.assertTrue(report["blocked"]) self.assertEqual( report["tiers"]["plan"]["capabilities"]["status"], "fail") def test_shadow_incapable_warns_only(self): report = dry_run_policy_document({ "mode": "shadow", "tiers": {"plan": {"model": "no-tools-model", "requires": {"tool_calling": True}}}, "model_capabilities": { "no-tools-model": {"tool_calling": False}, }, }) self.assertFalse(report["blocked"]) self.assertTrue(any("lacks" in w for w in report["warnings"])) def test_active_capable_does_not_block(self): # Audit fix 3（I1）：干跑新增网关模型检查；不传 # gateway_model 时 active 档 requires 无法验证即 # blocked。本用例的意图是“能力满足不拦”，所以 # 传一个同样满足的网关模型。 report = dry_run_policy_document({ "mode": "active", "tiers": {"plan": {"model": "plan-model", "requires": {"tool_calling": True}}}, "model_capabilities": { "plan-model": {"tool_calling": True}, }, }, gateway_model="plan-model") self.assertFalse(report["blocked"]) self.assertEqual( report["tiers"]["plan"]["capabilities"]["status"], "ok") def _completed_runner(cmd): if "dispatch" in cmd: name = cmd[cmd.index("--name") + 1] else: name = cmd[cmd.index("--task") + 1] return SimpleNamespace( returncode=0, stdout='{"task": "%s", "status": "completed", "text": "done"}' % name, stderr="") def _insert_task(db_path, task_id): now = time.time() conn = sqlite3.connect(db_path) try: conn.execute( "INSERT INTO tasks(id, project_id, prompt, status," " idempotency_key, depends_on, priority, created_by," " created_at, updated_at, last_heartbeat)" " VALUES(?, 'p1', 'do work', 'queued', NULL, '[]', 0," " 'tester', ?, ?, NULL)", (task_id, now, now)) conn.commit() finally: conn.close() class CapabilityWorkerGateTestCase(unittest.TestCase): def setUp(self): self.tmp = tempfile.TemporaryDirectory() self.db_path = os.path.join(self.tmp.name, "control.db") init_db(self.db_path) self.calls = [] def tearDown(self): self.tmp.cleanup() def _worker(self, policy_doc, model): def recording(cmd): self.calls.append(list(cmd)) return _completed_runner(cmd) # model_explicit=False: the worker model is a plain # default, not a CLI pin -- a CLI pin would bypass the # policy (and the capability check) by design, so the # gate test must not use one. return Worker(self.db_path, "/fake/host.py", runner=recording, env={"XCM_API_KEY": "test-placeholder"}, model=model, model_explicit=False, routing_policy=policy_doc) def test_blocked_decision_fails_task_without_running(self): _insert_task(self.db_path, "A") worker = self._worker({ "mode": "active", "tiers": {"default": {"model": "no-tools-model", "requires": {"tool_calling": True}}}, "model_capabilities": { "no-tools-model": {"tool_calling": False}, "small-model": {"tool_calling": True, "context_tokens": 32000}, }, }, model="small-model") # small-model is capable on tool calling but the task is # default-tier: the tier model fails tool calling, and the # worker default must be checked against the same # requires -- here it passes, so this is NOT a block. result = worker.run_one() self.assertEqual(result["status"], "completed") def test_blocked_decision_fails_task_when_worker_also_incapable(self): _insert_task(self.db_path, "B") worker = self._worker({ "mode": "active", "tiers": {"default": {"model": "no-tools-model", "requires": {"tool_calling": True}}}, "model_capabilities": { "no-tools-model": {"tool_calling": False}, "no-tools-worker": {"tool_calling": False}, }, }, model="no-tools-worker") result = worker.run_one() self.assertTrue(result["dispatched"]) self.assertEqual(result["status"], "failed") self.assertIn("tool_calling", result["error"]) self.assertEqual(self.calls, []) # gate fires before runner conn = sqlite3.connect(self.db_path) try: status = conn.execute( "SELECT status FROM tasks WHERE id = 'B'").fetchone()[0] finally: conn.close() self.assertEqual(status, "failed")