host
8d4771bce3
- crons: pause/resume, exponential-backoff retry (cap 1h), retry claim single-winner reusing quota+task build, run_cron_tick cap 5/project - swarm: per-agent task id + isolated workspace (resolve_inside gate), tool register + invocation audit (no execution in swarm layer) - presentation: worth-presenting rules (needs_input/failed/new content only) + notification ledger dedup 1h per (task, reason) - browser_tasks: one BrowserTransport instance per browser task, single-use (reuse after destroy refused), shots dir per instance - automations: paused status + retry columns (max_retries, retry_backoff_seconds, consecutive_failures, next_retry_at) - gap table docs/chv-muse-p8-gap.md; self-defined values listed there (not Meta official): tick cap 5, swarm max 32/default 8, retry base 60s cap 3600s, ledger window 3600s, snapshot trunc 200/64KB - tests: 49 new in cloud/tests/test_chv_muse_p8.py (incl. negatives: cron unauthorized trigger, cross-workspace access, no-content silence, dedup); full cloud suite 1916 OK (skipped=8)
700 lines
30 KiB
Python
700 lines
30 KiB
Python
"""对齐 Muse P8：后台工作（crons / 子代理群 / 回访通知 /
|
|
每任务浏览器子代理）单测。
|
|
|
|
不跑真实模型、不联网、不碰真机：临时 SQLite 文件 + 临时
|
|
state_dir + 模块函数直调 + fake transport/runner。
|
|
覆盖：cron 停用/恢复/重试退避/重试到期与抢单/tick 上限/
|
|
定义上限；群建群/派代理/工作区隔离/工具注册与调用/派发；
|
|
值得呈现评估/通知账本去重节流；浏览器实例单任务单实例/
|
|
用完即毁。
|
|
"""
|
|
|
|
import json
|
|
import os
|
|
import sqlite3
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
try:
|
|
from cloud.automations import cancel_automation
|
|
from cloud.automations import create_automation
|
|
from cloud.automations import ensure_automations_tables
|
|
from cloud.automations import get_automation
|
|
from cloud.automations import update_automation
|
|
from cloud.chv.browser_tasks import BrowserTransport
|
|
from cloud.chv.browser_tasks import create_instance
|
|
from cloud.chv.browser_tasks import destroy_instance
|
|
from cloud.chv.browser_tasks import get_instance
|
|
from cloud.chv.browser_tasks import instance_shots_dir
|
|
from cloud.chv.browser_tasks import ping_instance
|
|
from cloud.chv.crons import claim_retry_automation
|
|
from cloud.chv.crons import pause_automation
|
|
from cloud.chv.crons import record_automation_outcome
|
|
from cloud.chv.crons import resume_automation
|
|
from cloud.chv.crons import retry_delay_seconds
|
|
from cloud.chv.crons import retry_due_automations
|
|
from cloud.chv.crons import run_cron_tick
|
|
from cloud.chv.crons import run_retry_automation
|
|
from cloud.chv.presentation import NotificationLedger
|
|
from cloud.chv.presentation import evaluate_presentation
|
|
from cloud.chv.presentation import present_or_log
|
|
from cloud.chv.swarm import check_agent_path
|
|
from cloud.chv.swarm import create_swarm
|
|
from cloud.chv.swarm import dispatch_agent
|
|
from cloud.chv.swarm import get_swarm
|
|
from cloud.chv.swarm import list_agents
|
|
from cloud.chv.swarm import record_tool_invocation
|
|
from cloud.chv.swarm import register_tool
|
|
from cloud.chv.swarm import set_agent_status
|
|
from cloud.chv.swarm import spawn_agent
|
|
from cloud.server import init_db
|
|
except ImportError: # discover -s cloud/tests 时把项目根目录补进 sys.path
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
|
from cloud.automations import cancel_automation
|
|
from cloud.automations import create_automation
|
|
from cloud.automations import ensure_automations_tables
|
|
from cloud.automations import get_automation
|
|
from cloud.automations import update_automation
|
|
from cloud.chv.browser_tasks import BrowserTransport
|
|
from cloud.chv.browser_tasks import create_instance
|
|
from cloud.chv.browser_tasks import destroy_instance
|
|
from cloud.chv.browser_tasks import get_instance
|
|
from cloud.chv.browser_tasks import instance_shots_dir
|
|
from cloud.chv.browser_tasks import ping_instance
|
|
from cloud.chv.crons import claim_retry_automation
|
|
from cloud.chv.crons import pause_automation
|
|
from cloud.chv.crons import record_automation_outcome
|
|
from cloud.chv.crons import resume_automation
|
|
from cloud.chv.crons import retry_delay_seconds
|
|
from cloud.chv.crons import retry_due_automations
|
|
from cloud.chv.crons import run_cron_tick
|
|
from cloud.chv.crons import run_retry_automation
|
|
from cloud.chv.presentation import NotificationLedger
|
|
from cloud.chv.presentation import evaluate_presentation
|
|
from cloud.chv.presentation import present_or_log
|
|
from cloud.chv.swarm import check_agent_path
|
|
from cloud.chv.swarm import create_swarm
|
|
from cloud.chv.swarm import dispatch_agent
|
|
from cloud.chv.swarm import get_swarm
|
|
from cloud.chv.swarm import list_agents
|
|
from cloud.chv.swarm import record_tool_invocation
|
|
from cloud.chv.swarm import register_tool
|
|
from cloud.chv.swarm import set_agent_status
|
|
from cloud.chv.swarm import spawn_agent
|
|
from cloud.server import init_db
|
|
|
|
T0 = 1800000000.0
|
|
|
|
|
|
class CronTestCase(unittest.TestCase):
|
|
def setUp(self):
|
|
self.tmp = tempfile.TemporaryDirectory()
|
|
self.db_path = os.path.join(self.tmp.name, "control.db")
|
|
init_db(self.db_path)
|
|
self.conn = sqlite3.connect(self.db_path)
|
|
self.conn.row_factory = sqlite3.Row
|
|
ensure_automations_tables(self.conn)
|
|
|
|
def tearDown(self):
|
|
self.conn.close()
|
|
self.tmp.cleanup()
|
|
|
|
def _auto(self, name="a", **kwargs):
|
|
kwargs.setdefault("interval_seconds", 3600)
|
|
return create_automation(self.conn, "p1", name, "do thing",
|
|
"tester", now=T0, **kwargs)
|
|
|
|
def test_pause_resume_roundtrip(self):
|
|
auto = self._auto()
|
|
ok, paused = pause_automation(self.conn, auto["id"], now=T0)
|
|
self.assertTrue(ok)
|
|
self.assertEqual(paused["status"], "paused")
|
|
self.assertFalse(paused["due"])
|
|
ok, resumed = resume_automation(self.conn, auto["id"], now=T0)
|
|
self.assertTrue(ok)
|
|
self.assertEqual(resumed["status"], "active")
|
|
|
|
def test_pause_twice_and_resume_active_fail(self):
|
|
auto = self._auto()
|
|
self.assertTrue(pause_automation(self.conn, auto["id"],
|
|
now=T0)[0])
|
|
ok, _ = pause_automation(self.conn, auto["id"], now=T0)
|
|
self.assertFalse(ok)
|
|
ok, _ = resume_automation(self.conn, auto["id"], now=T0)
|
|
self.assertTrue(ok)
|
|
ok, _ = resume_automation(self.conn, auto["id"], now=T0)
|
|
self.assertFalse(ok)
|
|
|
|
def test_pause_missing_returns_false_none(self):
|
|
ok, auto = pause_automation(self.conn, "nope", now=T0)
|
|
self.assertFalse(ok)
|
|
self.assertIsNone(auto)
|
|
|
|
def test_cancel_paused_works(self):
|
|
auto = self._auto()
|
|
pause_automation(self.conn, auto["id"], now=T0)
|
|
ok, cancelled = cancel_automation(self.conn, auto["id"],
|
|
now=T0)
|
|
self.assertTrue(ok)
|
|
self.assertEqual(cancelled["status"], "cancelled")
|
|
|
|
def test_resume_name_clash_fails(self):
|
|
first = self._auto("same")
|
|
pause_automation(self.conn, first["id"], now=T0)
|
|
self._auto("same")
|
|
ok, _ = resume_automation(self.conn, first["id"], now=T0)
|
|
self.assertFalse(ok)
|
|
self.assertEqual(get_automation(self.conn, first["id"])
|
|
["status"], "paused")
|
|
|
|
def test_retry_delay_backoff_and_cap(self):
|
|
self.assertEqual(retry_delay_seconds(1, 60), 60)
|
|
self.assertEqual(retry_delay_seconds(2, 60), 120)
|
|
self.assertEqual(retry_delay_seconds(3, 60), 240)
|
|
self.assertEqual(retry_delay_seconds(0, 60), 60)
|
|
self.assertEqual(retry_delay_seconds(10, 60), 3600)
|
|
self.assertEqual(retry_delay_seconds(2, 3600), 3600)
|
|
|
|
def test_record_failure_schedules_retry(self):
|
|
auto = self._auto()
|
|
out = record_automation_outcome(self.conn, auto["id"],
|
|
"failed", now=T0)
|
|
self.assertEqual(out["consecutive_failures"], 1)
|
|
self.assertEqual(out["next_retry_at"], T0 + 60)
|
|
out = record_automation_outcome(self.conn, auto["id"],
|
|
"failed", now=T0 + 60)
|
|
self.assertEqual(out["consecutive_failures"], 2)
|
|
self.assertEqual(out["next_retry_at"], T0 + 60 + 120)
|
|
|
|
def test_record_success_clears_failures(self):
|
|
auto = self._auto()
|
|
record_automation_outcome(self.conn, auto["id"], "failed",
|
|
now=T0)
|
|
out = record_automation_outcome(self.conn, auto["id"],
|
|
"completed", now=T0 + 60)
|
|
self.assertEqual(out["consecutive_failures"], 0)
|
|
self.assertIsNone(out["next_retry_at"])
|
|
|
|
def test_skipped_quota_is_not_a_failure(self):
|
|
auto = self._auto()
|
|
record_automation_outcome(self.conn, auto["id"], "failed",
|
|
now=T0)
|
|
out = record_automation_outcome(self.conn, auto["id"],
|
|
"skipped_quota", now=T0 + 10)
|
|
self.assertEqual(out["consecutive_failures"], 1)
|
|
self.assertEqual(out["next_retry_at"], T0 + 60)
|
|
|
|
def test_retry_exhausted_after_max_retries(self):
|
|
auto = self._auto(max_retries=1)
|
|
out = record_automation_outcome(self.conn, auto["id"],
|
|
"failed", now=T0)
|
|
self.assertEqual(out["next_retry_at"], T0 + 60)
|
|
out = record_automation_outcome(self.conn, auto["id"],
|
|
"failed", now=T0 + 60)
|
|
self.assertEqual(out["consecutive_failures"], 2)
|
|
self.assertIsNone(out["next_retry_at"])
|
|
self.assertEqual(retry_due_automations(self.conn, "p1",
|
|
now=T0 + 61), [])
|
|
|
|
def test_retry_due_boundary(self):
|
|
auto = self._auto()
|
|
record_automation_outcome(self.conn, auto["id"], "failed",
|
|
now=T0)
|
|
self.assertEqual(retry_due_automations(self.conn, "p1",
|
|
now=T0 + 59), [])
|
|
due = retry_due_automations(self.conn, "p1", now=T0 + 60)
|
|
self.assertEqual([d["id"] for d in due], [auto["id"]])
|
|
|
|
def test_claim_retry_single_winner_and_clock_untouched(self):
|
|
auto = self._auto()
|
|
record_automation_outcome(self.conn, auto["id"], "failed",
|
|
now=T0)
|
|
before = get_automation(self.conn, auto["id"])
|
|
self.assertTrue(claim_retry_automation(self.conn, auto["id"],
|
|
now=T0 + 60))
|
|
self.assertFalse(claim_retry_automation(self.conn, auto["id"],
|
|
now=T0 + 60))
|
|
after = get_automation(self.conn, auto["id"])
|
|
self.assertEqual(after["last_run_at"],
|
|
before["last_run_at"])
|
|
self.assertIsNone(after["next_retry_at"])
|
|
|
|
def test_run_retry_automation_builds_task(self):
|
|
auto = self._auto()
|
|
record_automation_outcome(self.conn, auto["id"], "failed",
|
|
now=T0)
|
|
run = run_retry_automation(self.conn, auto["id"], now=T0 + 60)
|
|
self.assertIsNotNone(run)
|
|
self.assertEqual(run["status"], "created")
|
|
self.assertIsNotNone(run["task_id"])
|
|
self.assertIsNone(run_retry_automation(self.conn, auto["id"],
|
|
now=T0 + 60))
|
|
|
|
def test_run_retry_before_due_returns_none(self):
|
|
auto = self._auto()
|
|
record_automation_outcome(self.conn, auto["id"], "failed",
|
|
now=T0)
|
|
self.assertIsNone(run_retry_automation(self.conn, auto["id"],
|
|
now=T0 + 59))
|
|
|
|
def test_tick_caps_at_five_interval_first(self):
|
|
for i in range(4):
|
|
self._auto("due%d" % i, first_run_at=T0 - 10)
|
|
retry_ids = []
|
|
for i in range(2):
|
|
auto = self._auto("retry%d" % i,
|
|
first_run_at=T0 + 100000)
|
|
record_automation_outcome(self.conn, auto["id"], "failed",
|
|
now=T0)
|
|
retry_ids.append(auto["id"])
|
|
tick = run_cron_tick(self.conn, "p1", now=T0 + 61)
|
|
self.assertEqual(len(tick["due"]), 4)
|
|
self.assertEqual(len(tick["retry_due"]), 1)
|
|
self.assertTrue(tick["capped"])
|
|
# 两重试任务 next_retry_at 与 created_at 完全相同，
|
|
# 排序按 id 排，不按创建顺序；只断言属于其中之一。
|
|
self.assertIn(tick["retry_due"][0]["id"], retry_ids)
|
|
|
|
def test_tick_empty_when_nothing_due(self):
|
|
self._auto(first_run_at=T0 + 100000)
|
|
tick = run_cron_tick(self.conn, "p1", now=T0)
|
|
self.assertEqual(tick["due"], [])
|
|
self.assertEqual(tick["retry_due"], [])
|
|
self.assertFalse(tick["capped"])
|
|
|
|
def test_definition_cap_100(self):
|
|
for i in range(100):
|
|
self._auto("cap%d" % i)
|
|
with self.assertRaises(ValueError):
|
|
self._auto("cap100")
|
|
|
|
def test_update_retry_params(self):
|
|
auto = self._auto()
|
|
ok, updated = update_automation(
|
|
self.conn, auto["id"],
|
|
{"max_retries": 5, "retry_backoff_seconds": 120},
|
|
now=T0)
|
|
self.assertTrue(ok)
|
|
self.assertEqual(updated["max_retries"], 5)
|
|
self.assertEqual(updated["retry_backoff_seconds"], 120)
|
|
with self.assertRaises(ValueError):
|
|
update_automation(self.conn, auto["id"],
|
|
{"max_retries": 11}, now=T0)
|
|
with self.assertRaises(ValueError):
|
|
update_automation(self.conn, auto["id"],
|
|
{"retry_backoff_seconds": 5}, now=T0)
|
|
|
|
def test_create_retry_params_validated(self):
|
|
with self.assertRaises(ValueError):
|
|
self._auto("bad1", max_retries=11)
|
|
with self.assertRaises(ValueError):
|
|
self._auto("bad2", retry_backoff_seconds=4000)
|
|
auto = self._auto("ok", max_retries=0,
|
|
retry_backoff_seconds=10)
|
|
self.assertEqual(auto["max_retries"], 0)
|
|
|
|
|
|
class SwarmTestCase(unittest.TestCase):
|
|
def setUp(self):
|
|
self.tmp = tempfile.TemporaryDirectory()
|
|
self.state_dir = os.path.join(self.tmp.name, "state")
|
|
os.makedirs(self.state_dir)
|
|
self.root = os.path.join(self.tmp.name, "workspaces")
|
|
os.makedirs(self.root)
|
|
|
|
def tearDown(self):
|
|
self.tmp.cleanup()
|
|
|
|
def _swarm(self, max_agents=8):
|
|
return create_swarm(self.state_dir, "p1", "parent-1",
|
|
max_agents=max_agents, now=T0)
|
|
|
|
def test_create_swarm_defaults_and_caps(self):
|
|
swarm = self._swarm()
|
|
self.assertEqual(swarm["max_agents"], 8)
|
|
self.assertEqual(swarm["status"], "active")
|
|
with self.assertRaises(ValueError):
|
|
create_swarm(self.state_dir, "p1", "parent-1",
|
|
max_agents=0, now=T0)
|
|
with self.assertRaises(ValueError):
|
|
create_swarm(self.state_dir, "p1", "parent-1",
|
|
max_agents=33, now=T0)
|
|
|
|
def test_project_swarm_cap_10(self):
|
|
for _ in range(10):
|
|
self._swarm()
|
|
with self.assertRaises(ValueError):
|
|
self._swarm()
|
|
|
|
def test_spawn_agent_own_workspace(self):
|
|
swarm = self._swarm()
|
|
a1 = spawn_agent(self.state_dir, swarm["id"], "task one",
|
|
self.root, now=T0)
|
|
a2 = spawn_agent(self.state_dir, swarm["id"], "task two",
|
|
self.root, now=T0)
|
|
self.assertNotEqual(a1["task_id"], a2["task_id"])
|
|
self.assertNotEqual(a1["workspace"], a2["workspace"])
|
|
self.assertTrue(a1["workspace"].startswith(self.root))
|
|
self.assertEqual(a1["status"], "pending")
|
|
|
|
def test_spawn_beyond_max_agents_fails(self):
|
|
swarm = self._swarm(max_agents=1)
|
|
spawn_agent(self.state_dir, swarm["id"], "one", self.root,
|
|
now=T0)
|
|
with self.assertRaises(ValueError):
|
|
spawn_agent(self.state_dir, swarm["id"], "two", self.root,
|
|
now=T0)
|
|
|
|
def test_check_agent_path_isolation(self):
|
|
swarm = self._swarm()
|
|
a1 = spawn_agent(self.state_dir, swarm["id"], "one",
|
|
self.root, now=T0)
|
|
a2 = spawn_agent(self.state_dir, swarm["id"], "two",
|
|
self.root, now=T0)
|
|
self.assertTrue(check_agent_path(
|
|
self.state_dir, swarm["id"], a1["id"],
|
|
os.path.join(a1["workspace"], "out.txt")))
|
|
self.assertTrue(check_agent_path(
|
|
self.state_dir, swarm["id"], a1["id"], a1["workspace"]))
|
|
self.assertFalse(check_agent_path(
|
|
self.state_dir, swarm["id"], a1["id"], a2["workspace"]))
|
|
self.assertFalse(check_agent_path(
|
|
self.state_dir, swarm["id"], a1["id"], self.root))
|
|
self.assertFalse(check_agent_path(
|
|
self.state_dir, swarm["id"], a1["id"],
|
|
os.path.join(a1["workspace"], "..", "escape.txt")))
|
|
self.assertFalse(check_agent_path(
|
|
self.state_dir, swarm["id"], "nope", a1["workspace"]))
|
|
|
|
def test_swarm_status_derived_from_agents(self):
|
|
swarm = self._swarm()
|
|
a1 = spawn_agent(self.state_dir, swarm["id"], "one",
|
|
self.root, now=T0)
|
|
a2 = spawn_agent(self.state_dir, swarm["id"], "two",
|
|
self.root, now=T0)
|
|
set_agent_status(self.state_dir, swarm["id"], a1["id"],
|
|
"done", now=T0)
|
|
self.assertEqual(get_swarm(self.state_dir, swarm["id"])
|
|
["status"], "active")
|
|
set_agent_status(self.state_dir, swarm["id"], a2["id"],
|
|
"failed", now=T0)
|
|
self.assertEqual(get_swarm(self.state_dir, swarm["id"])
|
|
["status"], "failed")
|
|
with self.assertRaises(ValueError):
|
|
spawn_agent(self.state_dir, swarm["id"], "three",
|
|
self.root, now=T0)
|
|
|
|
def test_set_agent_status_invalid(self):
|
|
swarm = self._swarm()
|
|
agent = spawn_agent(self.state_dir, swarm["id"], "one",
|
|
self.root, now=T0)
|
|
with self.assertRaises(ValueError):
|
|
set_agent_status(self.state_dir, swarm["id"], agent["id"],
|
|
"nope", now=T0)
|
|
with self.assertRaises(ValueError):
|
|
set_agent_status(self.state_dir, swarm["id"], "nope",
|
|
"done", now=T0)
|
|
|
|
def test_list_agents_sorted(self):
|
|
swarm = self._swarm()
|
|
spawn_agent(self.state_dir, swarm["id"], "one", self.root,
|
|
now=T0)
|
|
spawn_agent(self.state_dir, swarm["id"], "two", self.root,
|
|
now=T0 + 1)
|
|
agents = list_agents(self.state_dir, swarm["id"])
|
|
self.assertEqual(len(agents), 2)
|
|
self.assertEqual(agents[0]["prompt"], "one")
|
|
with self.assertRaises(ValueError):
|
|
list_agents(self.state_dir, "nope")
|
|
|
|
def test_register_tool_namespaced_per_agent(self):
|
|
swarm = self._swarm()
|
|
a1 = spawn_agent(self.state_dir, swarm["id"], "one",
|
|
self.root, now=T0)
|
|
a2 = spawn_agent(self.state_dir, swarm["id"], "two",
|
|
self.root, now=T0)
|
|
tool = register_tool(self.state_dir, swarm["id"], a1["id"],
|
|
"search", "web search", now=T0)
|
|
self.assertEqual(tool["name"], "search")
|
|
with self.assertRaises(ValueError):
|
|
register_tool(self.state_dir, swarm["id"], a1["id"],
|
|
"search", "again", now=T0)
|
|
register_tool(self.state_dir, swarm["id"], a2["id"],
|
|
"search", "same name other agent", now=T0)
|
|
with self.assertRaises(ValueError):
|
|
register_tool(self.state_dir, swarm["id"], a1["id"],
|
|
"x" * 101, "too long", now=T0)
|
|
|
|
def test_record_tool_invocation_scoped(self):
|
|
swarm = self._swarm()
|
|
a1 = spawn_agent(self.state_dir, swarm["id"], "one",
|
|
self.root, now=T0)
|
|
a2 = spawn_agent(self.state_dir, swarm["id"], "two",
|
|
self.root, now=T0)
|
|
register_tool(self.state_dir, swarm["id"], a1["id"],
|
|
"search", "web search", now=T0)
|
|
record = record_tool_invocation(
|
|
self.state_dir, swarm["id"], a1["id"], "search",
|
|
"q=example", now=T0)
|
|
self.assertEqual(record["name"], "search")
|
|
with self.assertRaises(ValueError):
|
|
record_tool_invocation(self.state_dir, swarm["id"],
|
|
a2["id"], "search", "q=x", now=T0)
|
|
with self.assertRaises(ValueError):
|
|
record_tool_invocation(self.state_dir, swarm["id"],
|
|
a1["id"], "nosuch", "q=x", now=T0)
|
|
|
|
def test_dispatch_agent_runs_runner_once(self):
|
|
swarm = self._swarm()
|
|
agent = spawn_agent(self.state_dir, swarm["id"], "do it",
|
|
self.root, now=T0)
|
|
seen = []
|
|
|
|
def runner(task):
|
|
seen.append(task)
|
|
return {"status": "ok", "task_id": task["id"]}
|
|
|
|
result = dispatch_agent(self.state_dir, swarm["id"],
|
|
agent["id"], runner, now=T0)
|
|
self.assertEqual(result["status"], "ok")
|
|
self.assertEqual(seen[0]["id"], agent["task_id"])
|
|
self.assertEqual(seen[0]["workspace"], agent["workspace"])
|
|
with self.assertRaises(ValueError):
|
|
dispatch_agent(self.state_dir, swarm["id"], agent["id"],
|
|
runner, now=T0)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|
|
|
|
|
|
class PresentationTestCase(unittest.TestCase):
|
|
def setUp(self):
|
|
self.tmp = tempfile.TemporaryDirectory()
|
|
self.state_dir = os.path.join(self.tmp.name, "state")
|
|
os.makedirs(self.state_dir)
|
|
self.ledger = NotificationLedger(self.state_dir)
|
|
|
|
def tearDown(self):
|
|
self.tmp.cleanup()
|
|
|
|
def test_needs_input_and_failure_present(self):
|
|
self.assertEqual(
|
|
evaluate_presentation({"status": "needs_input"}),
|
|
{"present": True, "reason": "needs_input"})
|
|
self.assertEqual(
|
|
evaluate_presentation({"status": "failed"}),
|
|
{"present": True, "reason": "failed_needs_decision"})
|
|
self.assertEqual(
|
|
evaluate_presentation({"status": "interrupted"}),
|
|
{"present": True, "reason": "failed_needs_decision"})
|
|
|
|
def test_completed_with_new_content_presents(self):
|
|
self.assertTrue(evaluate_presentation(
|
|
{"status": "completed",
|
|
"new_artifacts": ["report.pdf"]})["present"])
|
|
self.assertTrue(evaluate_presentation(
|
|
{"status": "completed",
|
|
"changed_files": ["a.py"]})["present"])
|
|
self.assertTrue(evaluate_presentation(
|
|
{"status": "completed", "notify": True})["present"])
|
|
|
|
def test_routine_completed_and_cancelled_log_only(self):
|
|
self.assertEqual(
|
|
evaluate_presentation({"status": "completed"}),
|
|
{"present": False, "reason": "routine_completed"})
|
|
self.assertEqual(
|
|
evaluate_presentation({"status": "completed",
|
|
"new_artifacts": [],
|
|
"changed_files": []}),
|
|
{"present": False, "reason": "routine_completed"})
|
|
self.assertEqual(
|
|
evaluate_presentation({"status": "cancelled"}),
|
|
{"present": False, "reason": "cancelled"})
|
|
|
|
def test_unknown_status_fail_closed(self):
|
|
self.assertEqual(
|
|
evaluate_presentation({"status": "weird"}),
|
|
{"present": False, "reason": "unknown_status"})
|
|
self.assertEqual(
|
|
evaluate_presentation("not-a-dict"),
|
|
{"present": False, "reason": "unknown_status"})
|
|
|
|
def test_ledger_dedups_same_reason(self):
|
|
self.assertTrue(self.ledger.should_notify("t1", "failed",
|
|
now=T0))
|
|
self.assertFalse(self.ledger.should_notify("t1", "failed",
|
|
now=T0 + 10))
|
|
self.assertTrue(self.ledger.should_notify("t1", "needs_input",
|
|
now=T0 + 10))
|
|
self.assertTrue(self.ledger.should_notify("t2", "failed",
|
|
now=T0 + 10))
|
|
|
|
def test_ledger_window_expires(self):
|
|
self.assertTrue(self.ledger.should_notify("t1", "failed",
|
|
now=T0))
|
|
self.assertFalse(self.ledger.should_notify("t1", "failed",
|
|
now=T0 + 3599))
|
|
self.assertTrue(self.ledger.should_notify("t1", "failed",
|
|
now=T0 + 3600))
|
|
|
|
def test_ledger_rejects_empty_keys(self):
|
|
with self.assertRaises(ValueError):
|
|
self.ledger.should_notify("", "failed", now=T0)
|
|
with self.assertRaises(ValueError):
|
|
self.ledger.should_notify("t1", "", now=T0)
|
|
|
|
def test_ledger_corrupt_file_recovers(self):
|
|
self.assertTrue(self.ledger.should_notify("t1", "failed",
|
|
now=T0))
|
|
with open(os.path.join(self.state_dir,
|
|
"notification-ledger.json"),
|
|
"w") as fh:
|
|
fh.write("{not json")
|
|
self.assertTrue(self.ledger.should_notify("t2", "failed",
|
|
now=T0))
|
|
|
|
def test_present_or_log_combined(self):
|
|
out = present_or_log({"status": "failed"}, self.ledger,
|
|
"t1", now=T0)
|
|
self.assertEqual(out, {"present": True,
|
|
"reason": "failed_needs_decision",
|
|
"notified": True})
|
|
out = present_or_log({"status": "failed"}, self.ledger,
|
|
"t1", now=T0 + 5)
|
|
self.assertTrue(out["present"])
|
|
self.assertFalse(out["notified"])
|
|
out = present_or_log({"status": "completed"}, self.ledger,
|
|
"t2", now=T0)
|
|
self.assertEqual(out, {"present": False,
|
|
"reason": "routine_completed",
|
|
"notified": False})
|
|
|
|
|
|
class FakeTransport(BrowserTransport):
|
|
"""测试用 transport：记录 spawn/close，可注入失败。"""
|
|
|
|
def __init__(self):
|
|
self.spawned = []
|
|
self.closed = []
|
|
self.fail_spawn = False
|
|
self.fail_close = False
|
|
|
|
def spawn(self, instance):
|
|
if self.fail_spawn:
|
|
raise RuntimeError("spawn exploded")
|
|
self.spawned.append(instance["instance_id"])
|
|
|
|
def close(self, instance):
|
|
if self.fail_close:
|
|
raise RuntimeError("close exploded")
|
|
self.closed.append(instance["instance_id"])
|
|
|
|
def ping(self, instance):
|
|
return instance["instance_id"] in self.spawned \
|
|
and instance["instance_id"] not in self.closed
|
|
|
|
|
|
class BrowserTasksTestCase(unittest.TestCase):
|
|
def setUp(self):
|
|
self.tmp = tempfile.TemporaryDirectory()
|
|
self.state_dir = os.path.join(self.tmp.name, "state")
|
|
os.makedirs(self.state_dir)
|
|
self.transport = FakeTransport()
|
|
|
|
def tearDown(self):
|
|
self.tmp.cleanup()
|
|
|
|
def test_create_and_get_instance(self):
|
|
inst = create_instance(self.state_dir, "task-1",
|
|
self.transport, now=T0)
|
|
self.assertEqual(inst["status"], "active")
|
|
self.assertEqual(inst["task_id"], "task-1")
|
|
self.assertEqual(self.transport.spawned, [inst["instance_id"]])
|
|
got = get_instance(self.state_dir, "task-1")
|
|
self.assertEqual(got["instance_id"], inst["instance_id"])
|
|
self.assertIsNone(get_instance(self.state_dir, "nope"))
|
|
|
|
def test_two_tasks_get_distinct_instances(self):
|
|
a = create_instance(self.state_dir, "task-1", self.transport,
|
|
now=T0)
|
|
b = create_instance(self.state_dir, "task-2", self.transport,
|
|
now=T0)
|
|
self.assertNotEqual(a["instance_id"], b["instance_id"])
|
|
self.assertNotEqual(
|
|
instance_shots_dir(self.state_dir, "task-1"),
|
|
instance_shots_dir(self.state_dir, "task-2"))
|
|
|
|
def test_second_create_same_task_fails(self):
|
|
create_instance(self.state_dir, "task-1", self.transport,
|
|
now=T0)
|
|
with self.assertRaises(ValueError):
|
|
create_instance(self.state_dir, "task-1", self.transport,
|
|
now=T0)
|
|
|
|
def test_destroy_marks_destroyed_and_rejects_reuse(self):
|
|
inst = create_instance(self.state_dir, "task-1",
|
|
self.transport, now=T0)
|
|
destroyed = destroy_instance(self.state_dir, "task-1",
|
|
self.transport, now=T0 + 5)
|
|
self.assertEqual(destroyed["status"], "destroyed")
|
|
self.assertEqual(destroyed["destroyed_at"], T0 + 5)
|
|
self.assertEqual(self.transport.closed, [inst["instance_id"]])
|
|
with self.assertRaises(ValueError):
|
|
destroy_instance(self.state_dir, "task-1", self.transport,
|
|
now=T0 + 6)
|
|
with self.assertRaises(ValueError):
|
|
create_instance(self.state_dir, "task-1", self.transport,
|
|
now=T0 + 6)
|
|
self.assertFalse(ping_instance(self.state_dir, "task-1",
|
|
self.transport))
|
|
|
|
def test_destroy_missing_fails(self):
|
|
with self.assertRaises(ValueError):
|
|
destroy_instance(self.state_dir, "nope", self.transport,
|
|
now=T0)
|
|
|
|
def test_spawn_failure_records_failed(self):
|
|
self.transport.fail_spawn = True
|
|
with self.assertRaises(RuntimeError):
|
|
create_instance(self.state_dir, "task-1", self.transport,
|
|
now=T0)
|
|
got = get_instance(self.state_dir, "task-1")
|
|
self.assertEqual(got["status"], "failed")
|
|
|
|
def test_close_failure_still_destroys(self):
|
|
create_instance(self.state_dir, "task-1", self.transport,
|
|
now=T0)
|
|
self.transport.fail_close = True
|
|
destroyed = destroy_instance(self.state_dir, "task-1",
|
|
self.transport, now=T0)
|
|
self.assertEqual(destroyed["status"], "destroyed")
|
|
self.assertIn("close exploded", destroyed["close_error"])
|
|
|
|
def test_ping_active_and_missing(self):
|
|
self.assertFalse(ping_instance(self.state_dir, "task-1",
|
|
self.transport))
|
|
create_instance(self.state_dir, "task-1", self.transport,
|
|
now=T0)
|
|
self.assertTrue(ping_instance(self.state_dir, "task-1",
|
|
self.transport))
|
|
|
|
def test_shots_dir_inside_state_dir(self):
|
|
create_instance(self.state_dir, "task-1", self.transport,
|
|
now=T0)
|
|
shots = instance_shots_dir(self.state_dir, "task-1")
|
|
self.assertTrue(shots.startswith(self.state_dir))
|
|
self.assertIsNone(instance_shots_dir(self.state_dir, "nope"))
|
|
|
|
def test_base_transport_raises(self):
|
|
base = BrowserTransport()
|
|
with self.assertRaises(NotImplementedError):
|
|
base.spawn({})
|
|
with self.assertRaises(NotImplementedError):
|
|
base.close({})
|
|
with self.assertRaises(NotImplementedError):
|
|
base.ping({})
|