host
621c08d10f
B1: gateway capability check now triggers on walking the sub2api gateway, not on the gateway having filled a model; task-level explicit models are audited (worker_capability_gateway_explicit) but never blocked, preserving explicit priority; gateway-default models still fail-fast under active. I1: dry-run adds a gateway-model section, sourced only from CLOUD_SUB2API_DEFAULT_MODEL (never queries the relay); an active tier whose requires the gateway model fails or cannot verify blocks, shadow only warns. I2: shadow review gate now counts completed samples (completed >= 20), same basis for sample_shortfall; the endpoint attaches per-tier capability status (read-only policy load) and flags capability-failing tiers in the recommendation. I3: configured-but-broken routing policy logs a warning in load_routing_policy (unconfigured stays silent); the worker sets routing_policy_load_failed and audits worker_routing_policy_load_failed per task. I4: resume preamble wraps prior output/error as data with an explicit do-not-execute constraint; position verified in run_one (tree -> preamble -> project -> original) and the docs corrected from the false 'closest to the original prompt' claim. I5: decided model is written back with a CAS UPDATE (model IS NULL OR model=''), so later attempts reuse it; explicit models are never overwritten. I6: agent_* columns added to worker._ensure_result_columns and db.init_postgres_schema backfill; migrations backfills them for tasks before insert (sender_task_id pattern); column-level tests for agent_tool_approvals and task_attempts. I7: coordinator requeue adds AND status=<snapshot> (rowcount 0 skips requeue/audit) and filters paused_by_user / user_cancel / user_pause children, which are never revived or cleared. I8: explicit-model tasks (including coordinator children that inherited a parent model) audit a source=explicit worker_routing row; evaluate_shadow counts it as per-tier explicit_n only, never routed_n, never a dedup slot. M1/M2/M3/M5/M6/M8/M10: dedup slot moved after tier validation, shadow sort doc fixed, truncated recommendation notes truncation, seed cleanup on gateway/quota/goal-cap early returns, tools-only coordinator definitions rejected, preamble goal branch marked effectively unreachable, DESIGN.md drift corrected (outside repo). M4/M7/M9: documentation/new-field only (pass_rate_fully_measured, whitespace-model shadow-drop note, bound-note race note); old field semantics unchanged.
323 lines
13 KiB
Python
323 lines
13 KiB
Python
"""Capability routing (v8): directory, requires, decision filter,
|
|
worker capability gate, and dry-run reporting.
|
|
|
|
Run: python3 -m unittest discover -s cloud/tests -p "test_routing_capability.py" -v
|
|
"""
|
|
|
|
import os
|
|
import sqlite3
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
import unittest
|
|
from types import SimpleNamespace
|
|
|
|
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", ".."))
|
|
|
|
from cloud.models import (check_model_capabilities, decide_routing,
|
|
parse_model_capabilities, parse_routing_policy,
|
|
parse_tier_requires)
|
|
from cloud.routing_dryrun import dry_run_policy_document
|
|
from cloud.server import init_db
|
|
from cloud.worker import Worker
|
|
|
|
|
|
def _policy(**overrides):
|
|
doc = {
|
|
"version": "cap-v8",
|
|
"mode": "active",
|
|
"tiers": {
|
|
"plan": {"model": "plan-model",
|
|
"requires": {"tool_calling": True,
|
|
"min_context_tokens": 128000}},
|
|
},
|
|
"model_capabilities": {
|
|
"plan-model": {"tool_calling": True,
|
|
"context_tokens": 200000},
|
|
"worker-model": {"tool_calling": True,
|
|
"context_tokens": 200000},
|
|
"no-tools-model": {"tool_calling": False,
|
|
"context_tokens": 200000},
|
|
"small-model": {"tool_calling": True,
|
|
"context_tokens": 32000},
|
|
},
|
|
}
|
|
doc.update(overrides)
|
|
return parse_routing_policy(doc)
|
|
|
|
|
|
class CapabilityParseTestCase(unittest.TestCase):
|
|
def test_clean_capabilities_drops_bad_entries(self):
|
|
caps = parse_model_capabilities({
|
|
"good": {"tool_calling": True, "context_tokens": 1000},
|
|
"bad-bool": {"tool_calling": "yes"},
|
|
"bad-int": {"context_tokens": -5},
|
|
"": {"tool_calling": True},
|
|
"not-dict": "x",
|
|
})
|
|
self.assertEqual(list(caps.keys()), ["good"])
|
|
|
|
def test_requires_rejects_unknown_key_and_bad_value(self):
|
|
self.assertIsNone(parse_tier_requires({"fuzzy": True}))
|
|
self.assertIsNone(parse_tier_requires({"tool_calling": "yes"}))
|
|
self.assertIsNone(parse_tier_requires({"min_context_tokens": 0}))
|
|
self.assertEqual(parse_tier_requires({"tool_calling": False}), {})
|
|
self.assertEqual(
|
|
parse_tier_requires({"tool_calling": True,
|
|
"min_context_tokens": 100}),
|
|
{"tool_calling": True, "min_context_tokens": 100})
|
|
|
|
def test_invalid_requires_drops_only_that_tier(self):
|
|
doc = {
|
|
"mode": "active",
|
|
"tiers": {
|
|
"plan": {"model": "plan-model",
|
|
"requires": {"fuzzy": True}},
|
|
"default": "default-model",
|
|
},
|
|
}
|
|
policy = parse_routing_policy(doc)
|
|
self.assertNotIn("plan", policy.tiers)
|
|
self.assertIn("default", policy.tiers)
|
|
|
|
def test_check_matrix(self):
|
|
caps = {"m": {"tool_calling": True, "context_tokens": 1000}}
|
|
self.assertEqual(check_model_capabilities(caps, "m", {}),
|
|
(True, [], False))
|
|
self.assertEqual(check_model_capabilities(
|
|
caps, "m", {"tool_calling": True}), (True, [], False))
|
|
self.assertEqual(check_model_capabilities(
|
|
caps, "m", {"vision": True}), (False, ["vision"], True))
|
|
self.assertEqual(check_model_capabilities(
|
|
caps, "m", {"min_context_tokens": 2000}),
|
|
(False, ["min_context_tokens"], False))
|
|
self.assertEqual(check_model_capabilities(
|
|
caps, "absent", {"tool_calling": True}),
|
|
(False, ["tool_calling"], True))
|
|
|
|
|
|
class CapabilityDecisionTestCase(unittest.TestCase):
|
|
def test_active_capable_applies_policy_model(self):
|
|
decision = decide_routing(
|
|
{"id": "t1", "plan": True}, "worker-model",
|
|
policy=_policy())
|
|
self.assertEqual(decision.source, "policy")
|
|
self.assertEqual(decision.chosen_model, "plan-model")
|
|
self.assertTrue(decision.applied)
|
|
|
|
def test_active_incapable_falls_back_to_capable_worker_default(self):
|
|
policy = _policy()
|
|
policy.tiers["plan"] = "no-tools-model"
|
|
decision = decide_routing(
|
|
{"id": "t1", "plan": True}, "worker-model", policy=policy)
|
|
self.assertEqual(decision.source, "worker_default")
|
|
self.assertEqual(decision.chosen_model, "worker-model")
|
|
self.assertIn("lacks", decision.reason)
|
|
|
|
def test_active_incapable_and_worker_incapable_blocks(self):
|
|
policy = _policy()
|
|
policy.tiers["plan"] = "no-tools-model"
|
|
decision = decide_routing(
|
|
{"id": "t1", "plan": True}, "small-model", policy=policy)
|
|
self.assertEqual(decision.source, "capability_blocked")
|
|
self.assertIsNone(decision.chosen_model)
|
|
self.assertFalse(decision.applied)
|
|
# The reason names the tier model's shortfall (tool
|
|
# calling); the worker default's own shortfall (context)
|
|
# is what made the fallback fail, not the reason text.
|
|
self.assertIn("tool_calling", decision.reason)
|
|
|
|
def test_active_unknown_model_blocks(self):
|
|
policy = _policy()
|
|
policy.tiers["plan"] = "unlisted-model"
|
|
decision = decide_routing(
|
|
{"id": "t1", "plan": True}, "worker-model", policy=policy)
|
|
# worker default is capable, so it is used instead of blocking
|
|
self.assertEqual(decision.source, "worker_default")
|
|
self.assertEqual(decision.chosen_model, "worker-model")
|
|
self.assertIn("unknown", decision.reason)
|
|
|
|
def test_active_unknown_model_and_worker_unknown_blocks(self):
|
|
policy = _policy()
|
|
policy.tiers["plan"] = "unlisted-model"
|
|
decision = decide_routing(
|
|
{"id": "t1", "plan": True}, "also-unlisted", policy=policy)
|
|
self.assertEqual(decision.source, "capability_blocked")
|
|
self.assertIsNone(decision.chosen_model)
|
|
|
|
def test_shadow_incapable_never_blocks(self):
|
|
policy = parse_routing_policy({
|
|
"version": "cap-v8", "mode": "shadow",
|
|
"tiers": {"plan": {"model": "no-tools-model",
|
|
"requires": {"tool_calling": True}}},
|
|
"model_capabilities": {
|
|
"no-tools-model": {"tool_calling": False},
|
|
},
|
|
})
|
|
decision = decide_routing(
|
|
{"id": "t1", "plan": True}, "worker-model", policy=policy)
|
|
self.assertEqual(decision.source, "shadow")
|
|
self.assertEqual(decision.chosen_model, "worker-model")
|
|
self.assertEqual(decision.suggested_model, "no-tools-model")
|
|
self.assertIn("lacks", decision.reason)
|
|
|
|
def test_explicit_models_bypass_capability_check(self):
|
|
policy = _policy()
|
|
policy.tiers["plan"] = "no-tools-model"
|
|
decision = decide_routing(
|
|
{"id": "t1", "plan": True, "model": "no-tools-model"},
|
|
"small-model", policy=policy)
|
|
self.assertEqual(decision.source, "explicit")
|
|
self.assertEqual(decision.chosen_model, "no-tools-model")
|
|
decision = decide_routing(
|
|
{"id": "t1", "plan": True}, "no-tools-model",
|
|
policy=policy, worker_model_explicit=True)
|
|
self.assertEqual(decision.source, "worker_explicit")
|
|
self.assertEqual(decision.chosen_model, "no-tools-model")
|
|
|
|
def test_no_requirements_behaves_as_before(self):
|
|
policy = parse_routing_policy({
|
|
"mode": "active", "tiers": {"plan": "plan-model"}})
|
|
decision = decide_routing(
|
|
{"id": "t1", "plan": True}, "worker-model", policy=policy)
|
|
self.assertEqual(decision.source, "policy")
|
|
self.assertEqual(decision.chosen_model, "plan-model")
|
|
|
|
|
|
class CapabilityDryRunTestCase(unittest.TestCase):
|
|
def test_active_incapable_blocks_dry_run(self):
|
|
report = dry_run_policy_document({
|
|
"mode": "active",
|
|
"tiers": {"plan": {"model": "no-tools-model",
|
|
"requires": {"tool_calling": True}}},
|
|
"model_capabilities": {
|
|
"no-tools-model": {"tool_calling": False},
|
|
},
|
|
})
|
|
self.assertTrue(report["blocked"])
|
|
self.assertEqual(
|
|
report["tiers"]["plan"]["capabilities"]["status"], "fail")
|
|
|
|
def test_shadow_incapable_warns_only(self):
|
|
report = dry_run_policy_document({
|
|
"mode": "shadow",
|
|
"tiers": {"plan": {"model": "no-tools-model",
|
|
"requires": {"tool_calling": True}}},
|
|
"model_capabilities": {
|
|
"no-tools-model": {"tool_calling": False},
|
|
},
|
|
})
|
|
self.assertFalse(report["blocked"])
|
|
self.assertTrue(any("lacks" in w for w in report["warnings"]))
|
|
|
|
def test_active_capable_does_not_block(self):
|
|
# Audit fix 3（I1）：干跑新增网关模型检查；不传
|
|
# gateway_model 时 active 档 requires 无法验证即
|
|
# blocked。本用例的意图是“能力满足不拦”，所以
|
|
# 传一个同样满足的网关模型。
|
|
report = dry_run_policy_document({
|
|
"mode": "active",
|
|
"tiers": {"plan": {"model": "plan-model",
|
|
"requires": {"tool_calling": True}}},
|
|
"model_capabilities": {
|
|
"plan-model": {"tool_calling": True},
|
|
},
|
|
}, gateway_model="plan-model")
|
|
self.assertFalse(report["blocked"])
|
|
self.assertEqual(
|
|
report["tiers"]["plan"]["capabilities"]["status"], "ok")
|
|
|
|
|
|
def _completed_runner(cmd):
|
|
if "dispatch" in cmd:
|
|
name = cmd[cmd.index("--name") + 1]
|
|
else:
|
|
name = cmd[cmd.index("--task") + 1]
|
|
return SimpleNamespace(
|
|
returncode=0,
|
|
stdout='{"task": "%s", "status": "completed", "text": "done"}' % name,
|
|
stderr="")
|
|
|
|
|
|
def _insert_task(db_path, task_id):
|
|
now = time.time()
|
|
conn = sqlite3.connect(db_path)
|
|
try:
|
|
conn.execute(
|
|
"INSERT INTO tasks(id, project_id, prompt, status,"
|
|
" idempotency_key, depends_on, priority, created_by,"
|
|
" created_at, updated_at, last_heartbeat)"
|
|
" VALUES(?, 'p1', 'do work', 'queued', NULL, '[]', 0,"
|
|
" 'tester', ?, ?, NULL)", (task_id, now, now))
|
|
conn.commit()
|
|
finally:
|
|
conn.close()
|
|
|
|
|
|
class CapabilityWorkerGateTestCase(unittest.TestCase):
|
|
def setUp(self):
|
|
self.tmp = tempfile.TemporaryDirectory()
|
|
self.db_path = os.path.join(self.tmp.name, "control.db")
|
|
init_db(self.db_path)
|
|
self.calls = []
|
|
|
|
def tearDown(self):
|
|
self.tmp.cleanup()
|
|
|
|
def _worker(self, policy_doc, model):
|
|
def recording(cmd):
|
|
self.calls.append(list(cmd))
|
|
return _completed_runner(cmd)
|
|
|
|
# model_explicit=False: the worker model is a plain
|
|
# default, not a CLI pin -- a CLI pin would bypass the
|
|
# policy (and the capability check) by design, so the
|
|
# gate test must not use one.
|
|
return Worker(self.db_path, "/fake/host.py", runner=recording,
|
|
env={"XCM_API_KEY": "test-placeholder"},
|
|
model=model, model_explicit=False,
|
|
routing_policy=policy_doc)
|
|
|
|
def test_blocked_decision_fails_task_without_running(self):
|
|
_insert_task(self.db_path, "A")
|
|
worker = self._worker({
|
|
"mode": "active",
|
|
"tiers": {"default": {"model": "no-tools-model",
|
|
"requires": {"tool_calling": True}}},
|
|
"model_capabilities": {
|
|
"no-tools-model": {"tool_calling": False},
|
|
"small-model": {"tool_calling": True,
|
|
"context_tokens": 32000},
|
|
},
|
|
}, model="small-model")
|
|
# small-model is capable on tool calling but the task is
|
|
# default-tier: the tier model fails tool calling, and the
|
|
# worker default must be checked against the same
|
|
# requires -- here it passes, so this is NOT a block.
|
|
result = worker.run_one()
|
|
self.assertEqual(result["status"], "completed")
|
|
|
|
def test_blocked_decision_fails_task_when_worker_also_incapable(self):
|
|
_insert_task(self.db_path, "B")
|
|
worker = self._worker({
|
|
"mode": "active",
|
|
"tiers": {"default": {"model": "no-tools-model",
|
|
"requires": {"tool_calling": True}}},
|
|
"model_capabilities": {
|
|
"no-tools-model": {"tool_calling": False},
|
|
"no-tools-worker": {"tool_calling": False},
|
|
},
|
|
}, model="no-tools-worker")
|
|
result = worker.run_one()
|
|
self.assertTrue(result["dispatched"])
|
|
self.assertEqual(result["status"], "failed")
|
|
self.assertIn("tool_calling", result["error"])
|
|
self.assertEqual(self.calls, []) # gate fires before runner
|
|
conn = sqlite3.connect(self.db_path)
|
|
try:
|
|
status = conn.execute(
|
|
"SELECT status FROM tasks WHERE id = 'B'").fetchone()[0]
|
|
finally:
|
|
conn.close()
|
|
self.assertEqual(status, "failed")
|