host
d95d678379
N1-1: writeback scope verified too broad (any chosen, incl. shadow applied=False, worker_default and worker_explicit) and narrowed to source=policy + applied only; writeback now records a worker_model_persisted audit row as the identity marker. N1: persisted models are rechecked against current tier requires before dispatch on later attempts (fixes4 split known/unknown): active + known-missing fails fast (worker_capability_recheck_ rejected, no runner), unknown-only and shadow only audit. User-explicit, worker-CLI-explicit and gateway-filled tasks skip the recheck; legacy models are recognised by a policy applied worker_routing row. Recheck never rewrites tasks.model and never re-decides.
314 lines
12 KiB
Python
314 lines
12 KiB
Python
"""Audit fix 5: N1 persisted-model recheck (recheck + block).
|
|
|
|
- N1-1 (writeback scope): only source=policy + applied decisions are
|
|
persisted. Shadow (applied=False), worker_default (no policy / no
|
|
tier model / tier off / capability fallback) and worker_explicit
|
|
(CLI model) chosen models must NOT land in tasks.model, or the
|
|
next attempt would mistake them for a task-explicit model.
|
|
- N1 (recheck): a model persisted by I5 writeback is rechecked
|
|
against the CURRENT tier requires before dispatch on later
|
|
attempts. Active + known-missing fails fast (runner never
|
|
called); unknown-only audits only; shadow never blocks.
|
|
User-explicit models are untouched by the recheck, and
|
|
gateway-filled tasks skip it (the gateway check already
|
|
re-runs every attempt).
|
|
- Identification: writeback records worker_model_persisted;
|
|
pre-fix5 models are recognised by a legacy worker_routing
|
|
source=policy applied=true row with the same chosen model.
|
|
A source=explicit row alone proves nothing: the elif branch
|
|
writes one for every task that arrives with a model,
|
|
including persisted ones.
|
|
|
|
Run: python3 -m unittest discover -s cloud/tests -p "test_audit_fixes5.py" -v
|
|
No real models, no network, no real gateway: gateway steps are
|
|
stubbed exactly like test_audit_fixes.py / test_audit_fixes4.py.
|
|
"""
|
|
|
|
import json
|
|
import os
|
|
import sqlite3
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
import unittest
|
|
from types import SimpleNamespace
|
|
|
|
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", ".."))
|
|
|
|
from cloud.server import init_db
|
|
from cloud.worker import Worker
|
|
|
|
|
|
def _completed_runner(cmd):
|
|
if "dispatch" in cmd:
|
|
name = cmd[cmd.index("--name") + 1]
|
|
else:
|
|
name = cmd[cmd.index("--task") + 1]
|
|
return SimpleNamespace(
|
|
returncode=0,
|
|
stdout='{"status": "completed", "result": "done",'
|
|
' "task_id": "%s", "thread_id": "th1"}' % name,
|
|
stderr="")
|
|
|
|
|
|
def _insert_task(db_path, task_id, model=None, status="queued",
|
|
project_id="p1"):
|
|
now = time.time()
|
|
conn = sqlite3.connect(db_path)
|
|
try:
|
|
conn.execute(
|
|
"INSERT INTO tasks(id, project_id, prompt, status,"
|
|
" idempotency_key, depends_on, priority, created_by,"
|
|
" created_at, updated_at, last_heartbeat, model)"
|
|
" VALUES(?, ?, 'do work', ?, NULL, '[]', 0, 'tester',"
|
|
" ?, ?, NULL, ?)",
|
|
(task_id, project_id, status, now, now, model),
|
|
)
|
|
conn.commit()
|
|
finally:
|
|
conn.close()
|
|
|
|
|
|
def _insert_audit(db_path, project_id, action, detail):
|
|
conn = sqlite3.connect(db_path)
|
|
try:
|
|
# audit_log is created lazily by AuthStore on first audit;
|
|
# pre-seeding history must create it first.
|
|
conn.execute(
|
|
"CREATE TABLE IF NOT EXISTS audit_log("
|
|
"id INTEGER PRIMARY KEY AUTOINCREMENT, "
|
|
"user_id TEXT, project_id TEXT, action TEXT, "
|
|
"detail TEXT, created_at REAL)")
|
|
conn.commit()
|
|
conn.execute(
|
|
"INSERT INTO audit_log(user_id, project_id, action, detail,"
|
|
" created_at) VALUES('system', ?, ?, ?, ?)",
|
|
(project_id, action, detail, time.time()),
|
|
)
|
|
conn.commit()
|
|
finally:
|
|
conn.close()
|
|
|
|
|
|
def _row(db_path, task_id):
|
|
conn = sqlite3.connect(db_path)
|
|
conn.row_factory = sqlite3.Row
|
|
try:
|
|
row = conn.execute("SELECT * FROM tasks WHERE id = ?",
|
|
(task_id,)).fetchone()
|
|
return dict(row) if row else None
|
|
finally:
|
|
conn.close()
|
|
|
|
|
|
def _audit_actions(db_path, project_id="p1"):
|
|
from cloud.auth import AuthStore
|
|
store = AuthStore(db_path)
|
|
try:
|
|
return [(e["action"], e["detail"]) for e in store.list_audit(
|
|
project_id)]
|
|
finally:
|
|
store.conn.close()
|
|
|
|
|
|
def _worker(db_path, policy_doc=None, env=None, gateway=None,
|
|
model=None):
|
|
calls = []
|
|
|
|
def recording(cmd):
|
|
calls.append(list(cmd))
|
|
return _completed_runner(cmd)
|
|
|
|
worker = Worker(db_path, "/fake/host.py", runner=recording,
|
|
env=env or {"XCM_API_KEY": "test-placeholder"},
|
|
routing_policy=policy_doc, model=model)
|
|
if gateway is None:
|
|
worker._ensure_gateway = lambda task: (None, None)
|
|
else:
|
|
worker._ensure_gateway = lambda task: gateway
|
|
worker._sub2api_default_model = lambda e: "gateway-model"
|
|
worker._calls = calls
|
|
return worker
|
|
|
|
|
|
def _policy(mode, capabilities, requires=None):
|
|
return {
|
|
"mode": mode,
|
|
"tiers": {"default": {
|
|
"model": "policy-model",
|
|
"requires": requires if requires is not None
|
|
else {"tool_calling": True}}},
|
|
"model_capabilities": capabilities,
|
|
}
|
|
|
|
|
|
class WritebackScopeTestCase(unittest.TestCase):
|
|
def setUp(self):
|
|
self.tmp = tempfile.TemporaryDirectory()
|
|
self.db_path = os.path.join(self.tmp.name, "control.db")
|
|
init_db(self.db_path)
|
|
|
|
def tearDown(self):
|
|
self.tmp.cleanup()
|
|
|
|
def test_policy_applied_decision_is_persisted(self):
|
|
_insert_task(self.db_path, "T1")
|
|
worker = _worker(self.db_path, _policy("active", {
|
|
"policy-model": {"tool_calling": True},
|
|
}))
|
|
result = worker.run_one()
|
|
self.assertEqual(result["status"], "completed")
|
|
self.assertEqual(_row(self.db_path, "T1")["model"],
|
|
"policy-model")
|
|
persisted = [d for a, d in _audit_actions(self.db_path)
|
|
if a == "worker_model_persisted"]
|
|
self.assertTrue(persisted)
|
|
self.assertIn("model=policy-model", persisted[0])
|
|
|
|
def test_shadow_decision_is_not_persisted(self):
|
|
_insert_task(self.db_path, "T1")
|
|
worker = _worker(self.db_path, _policy("shadow", {
|
|
"policy-model": {"tool_calling": True},
|
|
}))
|
|
result = worker.run_one()
|
|
self.assertEqual(result["status"], "completed")
|
|
self.assertIsNone(_row(self.db_path, "T1")["model"])
|
|
self.assertFalse([a for a, _ in _audit_actions(self.db_path)
|
|
if a == "worker_model_persisted"])
|
|
|
|
def test_worker_default_decision_is_not_persisted(self):
|
|
_insert_task(self.db_path, "T1")
|
|
worker = _worker(self.db_path, None)
|
|
result = worker.run_one()
|
|
self.assertEqual(result["status"], "completed")
|
|
self.assertIsNone(_row(self.db_path, "T1")["model"])
|
|
self.assertFalse([a for a, _ in _audit_actions(self.db_path)
|
|
if a == "worker_model_persisted"])
|
|
|
|
def test_worker_explicit_decision_is_not_persisted(self):
|
|
_insert_task(self.db_path, "T1")
|
|
worker = _worker(self.db_path, None, model="cli-model")
|
|
result = worker.run_one()
|
|
self.assertEqual(result["status"], "completed")
|
|
self.assertIsNone(_row(self.db_path, "T1")["model"])
|
|
self.assertFalse([a for a, _ in _audit_actions(self.db_path)
|
|
if a == "worker_model_persisted"])
|
|
|
|
|
|
class RecheckTestCase(unittest.TestCase):
|
|
def setUp(self):
|
|
self.tmp = tempfile.TemporaryDirectory()
|
|
self.db_path = os.path.join(self.tmp.name, "control.db")
|
|
init_db(self.db_path)
|
|
|
|
def tearDown(self):
|
|
self.tmp.cleanup()
|
|
|
|
def _tightened(self, mode, capabilities):
|
|
return _policy(mode, capabilities)
|
|
|
|
def test_persisted_model_unchanged_requires_dispatches(self):
|
|
_insert_task(self.db_path, "T1", model="persisted-model")
|
|
_insert_audit(self.db_path, "p1", "worker_model_persisted",
|
|
"T1 tier=default model=persisted-model")
|
|
worker = _worker(self.db_path, self._tightened("active", {
|
|
"persisted-model": {"tool_calling": True},
|
|
}))
|
|
result = worker.run_one()
|
|
self.assertEqual(result["status"], "completed")
|
|
self.assertTrue(worker._calls)
|
|
self.assertFalse([a for a, _ in _audit_actions(self.db_path)
|
|
if a.startswith("worker_capability_recheck")])
|
|
|
|
def test_persisted_model_known_missing_is_blocked(self):
|
|
_insert_task(self.db_path, "T1", model="persisted-model")
|
|
_insert_audit(self.db_path, "p1", "worker_model_persisted",
|
|
"T1 tier=default model=persisted-model")
|
|
worker = _worker(self.db_path, self._tightened("active", {
|
|
"persisted-model": {"tool_calling": False},
|
|
}))
|
|
result = worker.run_one()
|
|
self.assertEqual(result["status"], "failed")
|
|
self.assertFalse(worker._calls)
|
|
self.assertIn("tool_calling", result["error"])
|
|
self.assertTrue([a for a, _ in _audit_actions(self.db_path)
|
|
if a == "worker_capability_recheck_rejected"])
|
|
# Recheck never rewrites the task model.
|
|
self.assertEqual(_row(self.db_path, "T1")["model"],
|
|
"persisted-model")
|
|
|
|
def test_legacy_policy_row_model_is_rechecked(self):
|
|
_insert_task(self.db_path, "T1", model="persisted-model")
|
|
_insert_audit(self.db_path, "p1", "worker_routing",
|
|
"T1 tier=default source=policy applied=true"
|
|
" suggested=- chosen=persisted-model")
|
|
worker = _worker(self.db_path, self._tightened("active", {
|
|
"persisted-model": {"tool_calling": False},
|
|
}))
|
|
result = worker.run_one()
|
|
self.assertEqual(result["status"], "failed")
|
|
self.assertFalse(worker._calls)
|
|
self.assertTrue([a for a, _ in _audit_actions(self.db_path)
|
|
if a == "worker_capability_recheck_rejected"])
|
|
|
|
def test_persisted_model_unknown_is_audited_not_blocked(self):
|
|
_insert_task(self.db_path, "T1", model="persisted-model")
|
|
_insert_audit(self.db_path, "p1", "worker_model_persisted",
|
|
"T1 tier=default model=persisted-model")
|
|
worker = _worker(self.db_path, self._tightened("active", {}))
|
|
result = worker.run_one()
|
|
self.assertEqual(result["status"], "completed")
|
|
self.assertTrue(worker._calls)
|
|
self.assertTrue([a for a, _ in _audit_actions(self.db_path)
|
|
if a == "worker_capability_recheck_unknown"])
|
|
self.assertFalse([a for a, _ in _audit_actions(self.db_path)
|
|
if a == "worker_capability_recheck_rejected"])
|
|
|
|
def test_persisted_model_shadow_known_missing_is_not_blocked(self):
|
|
_insert_task(self.db_path, "T1", model="persisted-model")
|
|
_insert_audit(self.db_path, "p1", "worker_model_persisted",
|
|
"T1 tier=default model=persisted-model")
|
|
worker = _worker(self.db_path, self._tightened("shadow", {
|
|
"persisted-model": {"tool_calling": False},
|
|
}))
|
|
result = worker.run_one()
|
|
self.assertEqual(result["status"], "completed")
|
|
self.assertTrue(worker._calls)
|
|
self.assertTrue([a for a, _ in _audit_actions(self.db_path)
|
|
if a == "worker_capability_recheck_shadow"])
|
|
self.assertFalse([a for a, _ in _audit_actions(self.db_path)
|
|
if a == "worker_capability_recheck_rejected"])
|
|
|
|
def test_user_explicit_model_is_not_rechecked(self):
|
|
# An explicit-only history must not be mistaken for a
|
|
# persisted model: the elif branch writes a source=explicit
|
|
# row for every task that arrives with a model, so that
|
|
# row alone proves nothing about origin.
|
|
_insert_task(self.db_path, "T1", model="explicit-model")
|
|
_insert_audit(self.db_path, "p1", "worker_routing",
|
|
"T1 tier=default source=explicit applied=false"
|
|
" suggested=- chosen=explicit-model")
|
|
worker = _worker(self.db_path, self._tightened("active", {
|
|
"explicit-model": {"tool_calling": False},
|
|
}))
|
|
result = worker.run_one()
|
|
self.assertEqual(result["status"], "completed")
|
|
self.assertTrue(worker._calls)
|
|
self.assertFalse([a for a, _ in _audit_actions(self.db_path)
|
|
if a.startswith("worker_capability_recheck")])
|
|
|
|
def test_gateway_filled_task_skips_recheck(self):
|
|
_insert_task(self.db_path, "T1")
|
|
worker = _worker(self.db_path, self._tightened("active", {
|
|
"gateway-model": {"tool_calling": True},
|
|
}), gateway=("sub2api", {}))
|
|
result = worker.run_one()
|
|
self.assertEqual(result["status"], "completed")
|
|
self.assertTrue(worker._calls)
|
|
self.assertFalse([a for a, _ in _audit_actions(self.db_path)
|
|
if a.startswith("worker_capability_recheck")])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|