host
e3423865aa
- M1: test_goal_branch_via_convert_queued_then_claim actually calls server._goal_convert_queued on a queued auto-resumed task before claiming, instead of only inserting the post-conversion state. - M5: test_worker_routing_audit_rows_carry_no_fingerprint drives a real run_one with an active policy and asserts worker_routing audit rows exist and carry no fingerprint (negative control verified: injecting fingerprint into the detail fails it). - M2: 'all callers' wording tightened to the real counts (load_routing_policy: 2 production call sites; evaluate_shadow: 1). - M4: content_fingerprint docstring now states the fingerprint covers the full normalized payload including fallback_models, which the router deliberately does not apply (fail-safe direction). No behaviour change; M3 DESIGN.md edits are outside the repo.
446 lines
18 KiB
Python
446 lines
18 KiB
Python
"""9.5-code track: code-reachable items only.
|
|
|
|
- Parse unification: agent_tools / tool_approvals column parsing
|
|
lives only in agent_def.py; server keeps None (undeclared),
|
|
worker maps None to [] / {} at the _task_from_row boundary.
|
|
- Goal branch of _apply_resume_preamble is reachable (via
|
|
_goal_convert_queued on a queued auto-resumed task), so it is
|
|
kept and tested, not deleted as dead code.
|
|
- Fail-open inventory: audit write failure and preamble lookup
|
|
failure log a warning without breaking dispatch.
|
|
- M7: shadow endpoint explicit_capability_failures_truncated flag.
|
|
- Policy content fingerprint: stable for the same document,
|
|
changes when requires change, never touches audit rows.
|
|
|
|
Run: python3 -m unittest discover -s cloud/tests -p "test_95_code.py" -v
|
|
No real models, no network, no real gateway, no real PG.
|
|
"""
|
|
|
|
import http.client
|
|
import json
|
|
import os
|
|
import sqlite3
|
|
import sys
|
|
import tempfile
|
|
import threading
|
|
import time
|
|
import unittest
|
|
from types import SimpleNamespace
|
|
from unittest import mock
|
|
|
|
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", ".."))
|
|
|
|
import cloud.worker as worker_module
|
|
from cloud.agent_def import (
|
|
parse_agent_tool_approvals_column,
|
|
parse_agent_tools_column,
|
|
)
|
|
from cloud.attempts import ensure_attempts_table, record_attempt
|
|
from cloud.auth import AuthStore
|
|
from cloud.models import parse_routing_policy
|
|
from cloud.routing_dryrun import dry_run_policy_document
|
|
from cloud.server import (
|
|
CloudControlPlaneHandler,
|
|
_agent_tool_approvals_from_row,
|
|
_agent_tools_from_row,
|
|
create_server,
|
|
init_db,
|
|
)
|
|
from cloud.worker import (
|
|
Worker,
|
|
_agent_tool_approvals_from_col,
|
|
_agent_tools_from_col,
|
|
)
|
|
|
|
|
|
class ParseUnificationTestCase(unittest.TestCase):
|
|
def test_shared_parser_is_single_source(self):
|
|
# Missing / dirty / wrong-type all parse to None.
|
|
for raw in (None, "not json", '{"a": 1}', '["a", 1]', 42):
|
|
if raw == '["a", 1]':
|
|
# A JSON list parses even if items are not strings;
|
|
# the parser only guarantees list shape, not items.
|
|
self.assertEqual(parse_agent_tools_column(raw),
|
|
["a", 1])
|
|
continue
|
|
self.assertIsNone(parse_agent_tools_column(raw))
|
|
for raw in (None, "not json", '["a"]', '{"a": 1}', 42):
|
|
if raw == '{"a": 1}':
|
|
self.assertEqual(
|
|
parse_agent_tool_approvals_column(raw), {"a": 1})
|
|
continue
|
|
self.assertIsNone(parse_agent_tool_approvals_column(raw))
|
|
# Valid values are copied, not aliased.
|
|
src = ["websearch"]
|
|
self.assertEqual(parse_agent_tools_column(src), src)
|
|
self.assertIsNot(parse_agent_tools_column(src), src)
|
|
src = {"websearch": "ask"}
|
|
self.assertEqual(parse_agent_tool_approvals_column(src), src)
|
|
self.assertIsNot(parse_agent_tool_approvals_column(src), src)
|
|
|
|
def test_server_keeps_none_worker_maps_to_empty(self):
|
|
# Server serialization: missing/dirty stays None (undeclared).
|
|
self.assertIsNone(_agent_tools_from_row({}))
|
|
self.assertIsNone(_agent_tools_from_row({"agent_tools": "junk"}))
|
|
self.assertEqual(
|
|
_agent_tools_from_row({"agent_tools": '["websearch"]'}),
|
|
["websearch"])
|
|
self.assertIsNone(_agent_tool_approvals_from_row({}))
|
|
self.assertIsNone(
|
|
_agent_tool_approvals_from_row(
|
|
{"agent_tool_approvals": "junk"}))
|
|
# Worker dispatch: same parse, None mapped to [] / {}.
|
|
self.assertEqual(_agent_tools_from_col(None), [])
|
|
self.assertEqual(_agent_tools_from_col("junk"), [])
|
|
self.assertEqual(_agent_tools_from_col('["websearch"]'),
|
|
["websearch"])
|
|
self.assertEqual(_agent_tool_approvals_from_col(None), {})
|
|
self.assertEqual(_agent_tool_approvals_from_col("junk"), {})
|
|
self.assertEqual(
|
|
_agent_tool_approvals_from_col('{"websearch": "ask"}'),
|
|
{"websearch": "ask"})
|
|
|
|
|
|
def _insert_task(db_path, task_id, prompt="做活", status="queued",
|
|
project_id="p1", **cols):
|
|
now = time.time()
|
|
conn = sqlite3.connect(db_path)
|
|
try:
|
|
base = {"id": task_id, "project_id": project_id,
|
|
"prompt": prompt, "status": status,
|
|
"idempotency_key": None, "depends_on": "[]",
|
|
"priority": 0, "created_by": "tester",
|
|
"created_at": now, "updated_at": now,
|
|
"last_heartbeat": None}
|
|
base.update(cols)
|
|
keys = list(base.keys())
|
|
conn.execute(
|
|
"INSERT INTO tasks(%s) VALUES(%s)"
|
|
% (", ".join(keys), ", ".join("?" * len(keys))),
|
|
[base[k] for k in keys])
|
|
conn.commit()
|
|
finally:
|
|
conn.close()
|
|
|
|
|
|
class WorkerCase(unittest.TestCase):
|
|
def setUp(self):
|
|
self.tmp = tempfile.TemporaryDirectory()
|
|
self.db_path = os.path.join(self.tmp.name, "control.db")
|
|
init_db(self.db_path)
|
|
self.env = {"XCM_API_KEY": "test-placeholder"}
|
|
|
|
def tearDown(self):
|
|
self.tmp.cleanup()
|
|
|
|
def _worker(self):
|
|
def _runner(cmd):
|
|
return SimpleNamespace(returncode=0, stdout="{}",
|
|
stderr="")
|
|
return Worker(self.db_path, "/fake/host.py",
|
|
runner=_runner, env=self.env)
|
|
|
|
|
|
class GoalBranchTestCase(WorkerCase):
|
|
def test_goal_branch_reached_for_converted_goal_task(self):
|
|
# Reachability proof (9.5-code): a non-goal task auto-resumed
|
|
# as platform_fresh (count=1, queued) that is then converted
|
|
# to a goal while still queued reaches the goal branch:
|
|
# no _resume (resume_message was cleared by auto_resume),
|
|
# no _goal_resume_breakpoint, count>0, latest attempt fresh.
|
|
_insert_task(self.db_path, "t1", prompt="做活",
|
|
goal="做目标", goal_mode=1, auto_resume_count=1)
|
|
conn = sqlite3.connect(self.db_path)
|
|
try:
|
|
ensure_attempts_table(conn)
|
|
record_attempt(conn, "t1", 1, "platform_fresh",
|
|
"worker_shutdown", -1,
|
|
"做到一半：已建 a.py", "被平台杀掉")
|
|
conn.commit()
|
|
finally:
|
|
conn.close()
|
|
worker = self._worker()
|
|
task = worker.get_task("t1")
|
|
injected = worker._apply_resume_preamble(task)
|
|
self.assertIn("上次尝试说明", injected["goal"])
|
|
self.assertTrue(injected["goal"].endswith("做目标"))
|
|
self.assertIn("上次尝试说明", injected["prompt"])
|
|
self.assertTrue(injected["prompt"].endswith("做活"))
|
|
# Memory only: the stored goal is untouched.
|
|
conn = sqlite3.connect(self.db_path)
|
|
try:
|
|
row = conn.execute("SELECT goal FROM tasks WHERE id = ?",
|
|
("t1",)).fetchone()
|
|
finally:
|
|
conn.close()
|
|
self.assertEqual(row[0], "做目标")
|
|
|
|
def test_goal_branch_via_convert_queued_then_claim(self):
|
|
# M1 strength (9.5-code minors): actually execute the
|
|
# conversion step itself, not just insert the
|
|
# post-conversion state. A queued non-goal task that was
|
|
# auto-resumed as platform_fresh is converted in place
|
|
# by server._goal_convert_queued, then claimed: the
|
|
# claim must reach the goal branch of
|
|
# _apply_resume_preamble (no _resume, no breakpoint).
|
|
_insert_task(self.db_path, "t2", prompt="做活",
|
|
auto_resume_count=1)
|
|
conn = sqlite3.connect(self.db_path)
|
|
try:
|
|
ensure_attempts_table(conn)
|
|
record_attempt(conn, "t2", 1, "platform_fresh",
|
|
"worker_shutdown", -1,
|
|
"做到一半：已建 a.py", "被平台杀掉")
|
|
conn.commit()
|
|
finally:
|
|
conn.close()
|
|
handler = CloudControlPlaneHandler.__new__(
|
|
CloudControlPlaneHandler)
|
|
handler.server = SimpleNamespace(db_path=self.db_path)
|
|
converted = handler._goal_convert_queued("t2", "做目标")
|
|
self.assertIsNotNone(converted)
|
|
self.assertEqual(converted["goal"], "做目标")
|
|
self.assertEqual(converted["goal_mode"], 1)
|
|
# Conversion is in-place on the queued row only: count
|
|
# and ledger untouched, no resume message planted.
|
|
conn = sqlite3.connect(self.db_path)
|
|
try:
|
|
row = conn.execute(
|
|
"SELECT status, goal_mode, auto_resume_count,"
|
|
" resume_message FROM tasks WHERE id = ?",
|
|
("t2",)).fetchone()
|
|
finally:
|
|
conn.close()
|
|
self.assertEqual(row[0], "queued")
|
|
self.assertEqual(row[1], 1)
|
|
self.assertEqual(row[2], 1)
|
|
self.assertIsNone(row[3])
|
|
worker = self._worker()
|
|
task = worker.get_task("t2")
|
|
self.assertFalse(task.get("_resume"))
|
|
self.assertFalse(task.get("_goal_resume_breakpoint"))
|
|
injected = worker._apply_resume_preamble(task)
|
|
self.assertIn("上次尝试说明", injected["goal"])
|
|
self.assertTrue(injected["goal"].endswith("做目标"))
|
|
self.assertIn("上次尝试说明", injected["prompt"])
|
|
self.assertTrue(injected["prompt"].endswith("做活"))
|
|
# Memory only: the stored goal is untouched.
|
|
conn = sqlite3.connect(self.db_path)
|
|
try:
|
|
row = conn.execute("SELECT goal FROM tasks WHERE id = ?",
|
|
("t2",)).fetchone()
|
|
finally:
|
|
conn.close()
|
|
self.assertEqual(row[0], "做目标")
|
|
|
|
|
|
class FailOpenLogTestCase(WorkerCase):
|
|
def test_audit_failure_logs_without_breaking_dispatch(self):
|
|
worker = self._worker()
|
|
calls = []
|
|
with mock.patch.object(
|
|
worker_module, "AuthStore",
|
|
side_effect=RuntimeError("db down")), \
|
|
mock.patch.object(
|
|
worker_module, "log_event",
|
|
side_effect=lambda *a, **k: calls.append(a[1])):
|
|
# Must not raise: audit is best-effort by design.
|
|
worker._audit("p1", "worker_claim", "t1")
|
|
self.assertEqual(calls, ["worker_audit_failed"])
|
|
|
|
def test_preamble_lookup_failure_logs_and_returns_task(self):
|
|
worker = self._worker()
|
|
task = {"id": "t1", "project_id": "p1", "prompt": "做活",
|
|
"auto_resume_count": 1}
|
|
calls = []
|
|
with mock.patch.object(
|
|
worker_module, "_db_connect",
|
|
side_effect=RuntimeError("db down")), \
|
|
mock.patch.object(
|
|
worker_module, "log_event",
|
|
side_effect=lambda *a, **k: calls.append(a[1])):
|
|
out = worker._apply_resume_preamble(task)
|
|
self.assertIs(out["prompt"], "做活")
|
|
self.assertEqual(
|
|
calls, ["worker_resume_preamble_lookup_failed"])
|
|
|
|
|
|
class FingerprintTestCase(unittest.TestCase):
|
|
def test_stable_and_sensitive_to_requires(self):
|
|
doc = {"mode": "shadow",
|
|
"tiers": {"default": "m1"},
|
|
"model_capabilities": {"m1": {"tool_calling": True}}}
|
|
a = parse_routing_policy(doc)
|
|
b = parse_routing_policy(
|
|
{"model_capabilities": {"m1": {"tool_calling": True}},
|
|
"tiers": {"default": "m1"}, "mode": "shadow"})
|
|
self.assertEqual(a.content_fingerprint(),
|
|
b.content_fingerprint())
|
|
c = parse_routing_policy({
|
|
"mode": "shadow",
|
|
"tiers": {"default": {"model": "m1",
|
|
"requires": {"tool_calling": True}}},
|
|
"model_capabilities": {"m1": {"tool_calling": True}}})
|
|
self.assertNotEqual(a.content_fingerprint(),
|
|
c.content_fingerprint())
|
|
# Operator version string does not affect the fingerprint.
|
|
d = parse_routing_policy(dict(doc, version="v99"))
|
|
self.assertEqual(a.content_fingerprint(),
|
|
d.content_fingerprint())
|
|
# Dry-run surfaces it; audit rows never carry it.
|
|
report = dry_run_policy_document(doc)
|
|
self.assertEqual(report["content_fingerprint"],
|
|
a.content_fingerprint())
|
|
|
|
|
|
def _completed_runner(cmd):
|
|
name = cmd[cmd.index("--name") + 1] if "dispatch" in cmd else "t"
|
|
return SimpleNamespace(
|
|
returncode=0,
|
|
stdout=json.dumps({"task": name, "status": "completed",
|
|
"text": "done"}),
|
|
stderr="")
|
|
|
|
|
|
class RoutingAuditNoFingerprintTestCase(unittest.TestCase):
|
|
# M5 (9.5-code minors): "fingerprint never enters audit rows"
|
|
# must be asserted on real audit rows, not just on the dry-run
|
|
# report. Drive a real run_one with an active policy so the
|
|
# worker writes a worker_routing audit row, then read the
|
|
# audit_log table directly. The test is sensitive: if anyone
|
|
# writes the fingerprint into a worker_routing detail, the
|
|
# "fingerprint" substring assertion fails; if routing stops
|
|
# writing audit rows at all, the "at least one row" assertion
|
|
# fails instead of passing vacuously.
|
|
def setUp(self):
|
|
self._old_push_allow = os.environ.get(
|
|
"CODEX_PUSH_ALLOW_PRIVATE_TARGETS")
|
|
os.environ["CODEX_PUSH_ALLOW_PRIVATE_TARGETS"] = "1"
|
|
self.tmp = tempfile.TemporaryDirectory()
|
|
self.db_path = os.path.join(self.tmp.name, "control.db")
|
|
init_db(self.db_path)
|
|
self.env = {"XCM_API_KEY": "test-placeholder"}
|
|
|
|
def tearDown(self):
|
|
if self._old_push_allow is None:
|
|
os.environ.pop("CODEX_PUSH_ALLOW_PRIVATE_TARGETS", None)
|
|
else:
|
|
os.environ["CODEX_PUSH_ALLOW_PRIVATE_TARGETS"] = (
|
|
self._old_push_allow)
|
|
self.tmp.cleanup()
|
|
|
|
def test_worker_routing_audit_rows_carry_no_fingerprint(self):
|
|
_insert_task(self.db_path, "t9", prompt="做活")
|
|
worker = Worker(
|
|
self.db_path, "/fake/host.py", runner=_completed_runner,
|
|
env=self.env, max_retries=0, retry_backoff_seconds=0,
|
|
routing_policy={"mode": "active",
|
|
"tiers": {"default": "m1"}})
|
|
out = worker.run_one()
|
|
self.assertEqual(out["status"], "completed")
|
|
conn = sqlite3.connect(self.db_path)
|
|
try:
|
|
rows = conn.execute(
|
|
"SELECT action, detail FROM audit_log").fetchall()
|
|
finally:
|
|
conn.close()
|
|
routing = [d for (a, d) in rows if a == "worker_routing"]
|
|
self.assertGreaterEqual(
|
|
len(routing), 1,
|
|
"expected at least one worker_routing audit row; "
|
|
"without it the fingerprint assertion would be vacuous")
|
|
for detail in routing:
|
|
self.assertNotIn("fingerprint", detail)
|
|
# The written format is exactly tier/source/applied/
|
|
# suggested/chosen; nothing else may ride along.
|
|
self.assertTrue(detail.startswith("t9 tier=default "))
|
|
for (_action, detail) in rows:
|
|
self.assertNotIn("fingerprint", detail or "")
|
|
|
|
|
|
class _HttpServerCase(unittest.TestCase):
|
|
def setUp(self):
|
|
self._old = {k: os.environ.get(k) for k in (
|
|
"CLOUD_ROUTING_POLICY_JSON", "CLOUD_ROUTING_POLICY_FILE")}
|
|
os.environ.pop("CLOUD_ROUTING_POLICY_JSON", None)
|
|
os.environ.pop("CLOUD_ROUTING_POLICY_FILE", None)
|
|
self.tmp = tempfile.TemporaryDirectory()
|
|
self.db_path = os.path.join(self.tmp.name, "control.db")
|
|
self.auth = AuthStore(self.db_path)
|
|
self.owner_id, self.owner_key = self.auth.create_user("owner")
|
|
self.server = create_server(self.db_path, "127.0.0.1", 0)
|
|
self.port = self.server.server_address[1]
|
|
self.thread = threading.Thread(
|
|
target=self.server.serve_forever, daemon=True)
|
|
self.thread.start()
|
|
|
|
def tearDown(self):
|
|
self.server.shutdown()
|
|
self.server.server_close()
|
|
self.thread.join(timeout=5)
|
|
for k, v in self._old.items():
|
|
if v is None:
|
|
os.environ.pop(k, None)
|
|
else:
|
|
os.environ[k] = v
|
|
try:
|
|
self.auth.conn.close()
|
|
except Exception:
|
|
pass
|
|
self.tmp.cleanup()
|
|
|
|
def _req(self, method, path, body=None, key=None):
|
|
conn = http.client.HTTPConnection(
|
|
"127.0.0.1", self.port, timeout=10)
|
|
hdrs = {}
|
|
payload = None
|
|
if body is not None:
|
|
payload = json.dumps(body).encode("utf-8")
|
|
hdrs["Content-Type"] = "application/json"
|
|
if key is not None:
|
|
hdrs["X-API-Key"] = key
|
|
try:
|
|
conn.request(method, path, body=payload, headers=hdrs)
|
|
resp = conn.getresponse()
|
|
data = resp.read()
|
|
finally:
|
|
conn.close()
|
|
try:
|
|
parsed = json.loads(data.decode("utf-8")) if data else None
|
|
except ValueError:
|
|
parsed = None
|
|
return resp.status, parsed
|
|
|
|
def _project(self, name="p"):
|
|
status, project = self._req(
|
|
"POST", "/projects", {"name": name, "auto_pr": 1},
|
|
self.owner_key)
|
|
self.assertEqual(status, 201, project)
|
|
return project["id"]
|
|
|
|
|
|
class ExplicitTruncationTestCase(_HttpServerCase):
|
|
def test_explicit_failures_truncated_flag(self):
|
|
pid = self._project()
|
|
conn = sqlite3.connect(self.db_path)
|
|
try:
|
|
conn.executemany(
|
|
"INSERT INTO audit_log(user_id, project_id, action,"
|
|
" detail, created_at) VALUES(?, ?, ?, ?, ?)",
|
|
[("tester", pid,
|
|
"worker_capability_gateway_explicit",
|
|
"E1 model=m1 tier=default mode=active"
|
|
" missing=tool_calling unknown=False blocked=True",
|
|
time.time())] * 10001)
|
|
conn.commit()
|
|
finally:
|
|
conn.close()
|
|
status, report = self._req(
|
|
"GET", "/routing-shadow?project_id=" + pid,
|
|
key=self.owner_key)
|
|
self.assertEqual(status, 200, report)
|
|
self.assertTrue(
|
|
report["explicit_capability_failures_truncated"])
|
|
self.assertEqual(
|
|
len(report["explicit_capability_failures"]), 10000)
|