"""Audit fix 5: N1 persisted-model recheck (recheck + block). - N1-1 (writeback scope): only source=policy + applied decisions are persisted. Shadow (applied=False), worker_default (no policy / no tier model / tier off / capability fallback) and worker_explicit (CLI model) chosen models must NOT land in tasks.model, or the next attempt would mistake them for a task-explicit model. - N1 (recheck): a model persisted by I5 writeback is rechecked against the CURRENT tier requires before dispatch on later attempts. Active + known-missing fails fast (runner never called); unknown-only audits only; shadow never blocks. User-explicit models are untouched by the recheck, and gateway-filled tasks skip it (the gateway check already re-runs every attempt). - Identification: writeback records worker_model_persisted; pre-fix5 models are recognised by a legacy worker_routing source=policy applied=true row with the same chosen model. A source=explicit row alone proves nothing: the elif branch writes one for every task that arrives with a model, including persisted ones. Run: python3 -m unittest discover -s cloud/tests -p "test_audit_fixes5.py" -v No real models, no network, no real gateway: gateway steps are stubbed exactly like test_audit_fixes.py / test_audit_fixes4.py. """ import json import os import sqlite3 import sys import tempfile import time import unittest from types import SimpleNamespace sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..")) from cloud.server import init_db from cloud.worker import Worker def _completed_runner(cmd): if "dispatch" in cmd: name = cmd[cmd.index("--name") + 1] else: name = cmd[cmd.index("--task") + 1] return SimpleNamespace( returncode=0, stdout='{"status": "completed", "result": "done",' ' "task_id": "%s", "thread_id": "th1"}' % name, stderr="") def _insert_task(db_path, task_id, model=None, status="queued", project_id="p1"): now = time.time() conn = sqlite3.connect(db_path) try: conn.execute( "INSERT INTO tasks(id, project_id, prompt, status," " idempotency_key, depends_on, priority, created_by," " created_at, updated_at, last_heartbeat, model)" " VALUES(?, ?, 'do work', ?, NULL, '[]', 0, 'tester'," " ?, ?, NULL, ?)", (task_id, project_id, status, now, now, model), ) conn.commit() finally: conn.close() def _insert_audit(db_path, project_id, action, detail): conn = sqlite3.connect(db_path) try: # audit_log is created lazily by AuthStore on first audit; # pre-seeding history must create it first. conn.execute( "CREATE TABLE IF NOT EXISTS audit_log(" "id INTEGER PRIMARY KEY AUTOINCREMENT, " "user_id TEXT, project_id TEXT, action TEXT, " "detail TEXT, created_at REAL)") conn.commit() conn.execute( "INSERT INTO audit_log(user_id, project_id, action, detail," " created_at) VALUES('system', ?, ?, ?, ?)", (project_id, action, detail, time.time()), ) conn.commit() finally: conn.close() def _row(db_path, task_id): conn = sqlite3.connect(db_path) conn.row_factory = sqlite3.Row try: row = conn.execute("SELECT * FROM tasks WHERE id = ?", (task_id,)).fetchone() return dict(row) if row else None finally: conn.close() def _audit_actions(db_path, project_id="p1"): from cloud.auth import AuthStore store = AuthStore(db_path) try: return [(e["action"], e["detail"]) for e in store.list_audit( project_id)] finally: store.conn.close() def _worker(db_path, policy_doc=None, env=None, gateway=None, model=None): calls = [] def recording(cmd): calls.append(list(cmd)) return _completed_runner(cmd) worker = Worker(db_path, "/fake/host.py", runner=recording, env=env or {"XCM_API_KEY": "test-placeholder"}, routing_policy=policy_doc, model=model) if gateway is None: worker._ensure_gateway = lambda task: (None, None) else: worker._ensure_gateway = lambda task: gateway worker._sub2api_default_model = lambda e: "gateway-model" worker._calls = calls return worker def _policy(mode, capabilities, requires=None): return { "mode": mode, "tiers": {"default": { "model": "policy-model", "requires": requires if requires is not None else {"tool_calling": True}}}, "model_capabilities": capabilities, } class WritebackScopeTestCase(unittest.TestCase): def setUp(self): self.tmp = tempfile.TemporaryDirectory() self.db_path = os.path.join(self.tmp.name, "control.db") init_db(self.db_path) def tearDown(self): self.tmp.cleanup() def test_policy_applied_decision_is_persisted(self): _insert_task(self.db_path, "T1") worker = _worker(self.db_path, _policy("active", { "policy-model": {"tool_calling": True}, })) result = worker.run_one() self.assertEqual(result["status"], "completed") self.assertEqual(_row(self.db_path, "T1")["model"], "policy-model") persisted = [d for a, d in _audit_actions(self.db_path) if a == "worker_model_persisted"] self.assertTrue(persisted) self.assertIn("model=policy-model", persisted[0]) def test_shadow_decision_is_not_persisted(self): _insert_task(self.db_path, "T1") worker = _worker(self.db_path, _policy("shadow", { "policy-model": {"tool_calling": True}, })) result = worker.run_one() self.assertEqual(result["status"], "completed") self.assertIsNone(_row(self.db_path, "T1")["model"]) self.assertFalse([a for a, _ in _audit_actions(self.db_path) if a == "worker_model_persisted"]) def test_worker_default_decision_is_not_persisted(self): _insert_task(self.db_path, "T1") worker = _worker(self.db_path, None) result = worker.run_one() self.assertEqual(result["status"], "completed") self.assertIsNone(_row(self.db_path, "T1")["model"]) self.assertFalse([a for a, _ in _audit_actions(self.db_path) if a == "worker_model_persisted"]) def test_worker_explicit_decision_is_not_persisted(self): _insert_task(self.db_path, "T1") worker = _worker(self.db_path, None, model="cli-model") result = worker.run_one() self.assertEqual(result["status"], "completed") self.assertIsNone(_row(self.db_path, "T1")["model"]) self.assertFalse([a for a, _ in _audit_actions(self.db_path) if a == "worker_model_persisted"]) class RecheckTestCase(unittest.TestCase): def setUp(self): self.tmp = tempfile.TemporaryDirectory() self.db_path = os.path.join(self.tmp.name, "control.db") init_db(self.db_path) def tearDown(self): self.tmp.cleanup() def _tightened(self, mode, capabilities): return _policy(mode, capabilities) def test_persisted_model_unchanged_requires_dispatches(self): _insert_task(self.db_path, "T1", model="persisted-model") _insert_audit(self.db_path, "p1", "worker_model_persisted", "T1 tier=default model=persisted-model") worker = _worker(self.db_path, self._tightened("active", { "persisted-model": {"tool_calling": True}, })) result = worker.run_one() self.assertEqual(result["status"], "completed") self.assertTrue(worker._calls) self.assertFalse([a for a, _ in _audit_actions(self.db_path) if a.startswith("worker_capability_recheck")]) def test_persisted_model_known_missing_is_blocked(self): _insert_task(self.db_path, "T1", model="persisted-model") _insert_audit(self.db_path, "p1", "worker_model_persisted", "T1 tier=default model=persisted-model") worker = _worker(self.db_path, self._tightened("active", { "persisted-model": {"tool_calling": False}, })) result = worker.run_one() self.assertEqual(result["status"], "failed") self.assertFalse(worker._calls) self.assertIn("tool_calling", result["error"]) self.assertTrue([a for a, _ in _audit_actions(self.db_path) if a == "worker_capability_recheck_rejected"]) # Recheck never rewrites the task model. self.assertEqual(_row(self.db_path, "T1")["model"], "persisted-model") def test_legacy_policy_row_model_is_rechecked(self): _insert_task(self.db_path, "T1", model="persisted-model") _insert_audit(self.db_path, "p1", "worker_routing", "T1 tier=default source=policy applied=true" " suggested=- chosen=persisted-model") worker = _worker(self.db_path, self._tightened("active", { "persisted-model": {"tool_calling": False}, })) result = worker.run_one() self.assertEqual(result["status"], "failed") self.assertFalse(worker._calls) self.assertTrue([a for a, _ in _audit_actions(self.db_path) if a == "worker_capability_recheck_rejected"]) def test_persisted_model_unknown_is_audited_not_blocked(self): _insert_task(self.db_path, "T1", model="persisted-model") _insert_audit(self.db_path, "p1", "worker_model_persisted", "T1 tier=default model=persisted-model") worker = _worker(self.db_path, self._tightened("active", {})) result = worker.run_one() self.assertEqual(result["status"], "completed") self.assertTrue(worker._calls) self.assertTrue([a for a, _ in _audit_actions(self.db_path) if a == "worker_capability_recheck_unknown"]) self.assertFalse([a for a, _ in _audit_actions(self.db_path) if a == "worker_capability_recheck_rejected"]) def test_persisted_model_shadow_known_missing_is_not_blocked(self): _insert_task(self.db_path, "T1", model="persisted-model") _insert_audit(self.db_path, "p1", "worker_model_persisted", "T1 tier=default model=persisted-model") worker = _worker(self.db_path, self._tightened("shadow", { "persisted-model": {"tool_calling": False}, })) result = worker.run_one() self.assertEqual(result["status"], "completed") self.assertTrue(worker._calls) self.assertTrue([a for a, _ in _audit_actions(self.db_path) if a == "worker_capability_recheck_shadow"]) self.assertFalse([a for a, _ in _audit_actions(self.db_path) if a == "worker_capability_recheck_rejected"]) def test_user_explicit_model_is_not_rechecked(self): # An explicit-only history must not be mistaken for a # persisted model: the elif branch writes a source=explicit # row for every task that arrives with a model, so that # row alone proves nothing about origin. _insert_task(self.db_path, "T1", model="explicit-model") _insert_audit(self.db_path, "p1", "worker_routing", "T1 tier=default source=explicit applied=false" " suggested=- chosen=explicit-model") worker = _worker(self.db_path, self._tightened("active", { "explicit-model": {"tool_calling": False}, })) result = worker.run_one() self.assertEqual(result["status"], "completed") self.assertTrue(worker._calls) self.assertFalse([a for a, _ in _audit_actions(self.db_path) if a.startswith("worker_capability_recheck")]) def test_gateway_filled_task_skips_recheck(self): _insert_task(self.db_path, "T1") worker = _worker(self.db_path, self._tightened("active", { "gateway-model": {"tool_calling": True}, }), gateway=("sub2api", {})) result = worker.run_one() self.assertEqual(result["status"], "completed") self.assertTrue(worker._calls) self.assertFalse([a for a, _ in _audit_actions(self.db_path) if a.startswith("worker_capability_recheck")]) if __name__ == "__main__": unittest.main()