merge(chv-muse-round12): 合入 P8 后台工作 (8d4771b)

cloud/tests/test_chv_muse_p8.py

@@ -0,0 +1,699 @@
+"""对齐 Muse P8：后台工作（crons / 子代理群 / 回访通知 /
+每任务浏览器子代理）单测。
+
+不跑真实模型、不联网、不碰真机：临时 SQLite 文件 + 临时
+state_dir + 模块函数直调 + fake transport/runner。
+覆盖：cron 停用/恢复/重试退避/重试到期与抢单/tick 上限/
+定义上限；群建群/派代理/工作区隔离/工具注册与调用/派发；
+值得呈现评估/通知账本去重节流；浏览器实例单任务单实例/
+用完即毁。
+"""
+
+import json
+import os
+import sqlite3
+import sys
+import tempfile
+import time
+import unittest
+from pathlib import Path
+
+try:
+    from cloud.automations import cancel_automation
+    from cloud.automations import create_automation
+    from cloud.automations import ensure_automations_tables
+    from cloud.automations import get_automation
+    from cloud.automations import update_automation
+    from cloud.chv.browser_tasks import BrowserTransport
+    from cloud.chv.browser_tasks import create_instance
+    from cloud.chv.browser_tasks import destroy_instance
+    from cloud.chv.browser_tasks import get_instance
+    from cloud.chv.browser_tasks import instance_shots_dir
+    from cloud.chv.browser_tasks import ping_instance
+    from cloud.chv.crons import claim_retry_automation
+    from cloud.chv.crons import pause_automation
+    from cloud.chv.crons import record_automation_outcome
+    from cloud.chv.crons import resume_automation
+    from cloud.chv.crons import retry_delay_seconds
+    from cloud.chv.crons import retry_due_automations
+    from cloud.chv.crons import run_cron_tick
+    from cloud.chv.crons import run_retry_automation
+    from cloud.chv.presentation import NotificationLedger
+    from cloud.chv.presentation import evaluate_presentation
+    from cloud.chv.presentation import present_or_log
+    from cloud.chv.swarm import check_agent_path
+    from cloud.chv.swarm import create_swarm
+    from cloud.chv.swarm import dispatch_agent
+    from cloud.chv.swarm import get_swarm
+    from cloud.chv.swarm import list_agents
+    from cloud.chv.swarm import record_tool_invocation
+    from cloud.chv.swarm import register_tool
+    from cloud.chv.swarm import set_agent_status
+    from cloud.chv.swarm import spawn_agent
+    from cloud.server import init_db
+except ImportError:  # discover -s cloud/tests 时把项目根目录补进 sys.path
+    sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
+    from cloud.automations import cancel_automation
+    from cloud.automations import create_automation
+    from cloud.automations import ensure_automations_tables
+    from cloud.automations import get_automation
+    from cloud.automations import update_automation
+    from cloud.chv.browser_tasks import BrowserTransport
+    from cloud.chv.browser_tasks import create_instance
+    from cloud.chv.browser_tasks import destroy_instance
+    from cloud.chv.browser_tasks import get_instance
+    from cloud.chv.browser_tasks import instance_shots_dir
+    from cloud.chv.browser_tasks import ping_instance
+    from cloud.chv.crons import claim_retry_automation
+    from cloud.chv.crons import pause_automation
+    from cloud.chv.crons import record_automation_outcome
+    from cloud.chv.crons import resume_automation
+    from cloud.chv.crons import retry_delay_seconds
+    from cloud.chv.crons import retry_due_automations
+    from cloud.chv.crons import run_cron_tick
+    from cloud.chv.crons import run_retry_automation
+    from cloud.chv.presentation import NotificationLedger
+    from cloud.chv.presentation import evaluate_presentation
+    from cloud.chv.presentation import present_or_log
+    from cloud.chv.swarm import check_agent_path
+    from cloud.chv.swarm import create_swarm
+    from cloud.chv.swarm import dispatch_agent
+    from cloud.chv.swarm import get_swarm
+    from cloud.chv.swarm import list_agents
+    from cloud.chv.swarm import record_tool_invocation
+    from cloud.chv.swarm import register_tool
+    from cloud.chv.swarm import set_agent_status
+    from cloud.chv.swarm import spawn_agent
+    from cloud.server import init_db
+
+T0 = 1800000000.0
+
+
+class CronTestCase(unittest.TestCase):
+    def setUp(self):
+        self.tmp = tempfile.TemporaryDirectory()
+        self.db_path = os.path.join(self.tmp.name, "control.db")
+        init_db(self.db_path)
+        self.conn = sqlite3.connect(self.db_path)
+        self.conn.row_factory = sqlite3.Row
+        ensure_automations_tables(self.conn)
+
+    def tearDown(self):
+        self.conn.close()
+        self.tmp.cleanup()
+
+    def _auto(self, name="a", **kwargs):
+        kwargs.setdefault("interval_seconds", 3600)
+        return create_automation(self.conn, "p1", name, "do thing",
+                                 "tester", now=T0, **kwargs)
+
+    def test_pause_resume_roundtrip(self):
+        auto = self._auto()
+        ok, paused = pause_automation(self.conn, auto["id"], now=T0)
+        self.assertTrue(ok)
+        self.assertEqual(paused["status"], "paused")
+        self.assertFalse(paused["due"])
+        ok, resumed = resume_automation(self.conn, auto["id"], now=T0)
+        self.assertTrue(ok)
+        self.assertEqual(resumed["status"], "active")
+
+    def test_pause_twice_and_resume_active_fail(self):
+        auto = self._auto()
+        self.assertTrue(pause_automation(self.conn, auto["id"],
+                                         now=T0)[0])
+        ok, _ = pause_automation(self.conn, auto["id"], now=T0)
+        self.assertFalse(ok)
+        ok, _ = resume_automation(self.conn, auto["id"], now=T0)
+        self.assertTrue(ok)
+        ok, _ = resume_automation(self.conn, auto["id"], now=T0)
+        self.assertFalse(ok)
+
+    def test_pause_missing_returns_false_none(self):
+        ok, auto = pause_automation(self.conn, "nope", now=T0)
+        self.assertFalse(ok)
+        self.assertIsNone(auto)
+
+    def test_cancel_paused_works(self):
+        auto = self._auto()
+        pause_automation(self.conn, auto["id"], now=T0)
+        ok, cancelled = cancel_automation(self.conn, auto["id"],
+                                          now=T0)
+        self.assertTrue(ok)
+        self.assertEqual(cancelled["status"], "cancelled")
+
+    def test_resume_name_clash_fails(self):
+        first = self._auto("same")
+        pause_automation(self.conn, first["id"], now=T0)
+        self._auto("same")
+        ok, _ = resume_automation(self.conn, first["id"], now=T0)
+        self.assertFalse(ok)
+        self.assertEqual(get_automation(self.conn, first["id"])
+                         ["status"], "paused")
+
+    def test_retry_delay_backoff_and_cap(self):
+        self.assertEqual(retry_delay_seconds(1, 60), 60)
+        self.assertEqual(retry_delay_seconds(2, 60), 120)
+        self.assertEqual(retry_delay_seconds(3, 60), 240)
+        self.assertEqual(retry_delay_seconds(0, 60), 60)
+        self.assertEqual(retry_delay_seconds(10, 60), 3600)
+        self.assertEqual(retry_delay_seconds(2, 3600), 3600)
+
+    def test_record_failure_schedules_retry(self):
+        auto = self._auto()
+        out = record_automation_outcome(self.conn, auto["id"],
+                                        "failed", now=T0)
+        self.assertEqual(out["consecutive_failures"], 1)
+        self.assertEqual(out["next_retry_at"], T0 + 60)
+        out = record_automation_outcome(self.conn, auto["id"],
+                                        "failed", now=T0 + 60)
+        self.assertEqual(out["consecutive_failures"], 2)
+        self.assertEqual(out["next_retry_at"], T0 + 60 + 120)
+
+    def test_record_success_clears_failures(self):
+        auto = self._auto()
+        record_automation_outcome(self.conn, auto["id"], "failed",
+                                  now=T0)
+        out = record_automation_outcome(self.conn, auto["id"],
+                                        "completed", now=T0 + 60)
+        self.assertEqual(out["consecutive_failures"], 0)
+        self.assertIsNone(out["next_retry_at"])
+
+    def test_skipped_quota_is_not_a_failure(self):
+        auto = self._auto()
+        record_automation_outcome(self.conn, auto["id"], "failed",
+                                  now=T0)
+        out = record_automation_outcome(self.conn, auto["id"],
+                                        "skipped_quota", now=T0 + 10)
+        self.assertEqual(out["consecutive_failures"], 1)
+        self.assertEqual(out["next_retry_at"], T0 + 60)
+
+    def test_retry_exhausted_after_max_retries(self):
+        auto = self._auto(max_retries=1)
+        out = record_automation_outcome(self.conn, auto["id"],
+                                        "failed", now=T0)
+        self.assertEqual(out["next_retry_at"], T0 + 60)
+        out = record_automation_outcome(self.conn, auto["id"],
+                                        "failed", now=T0 + 60)
+        self.assertEqual(out["consecutive_failures"], 2)
+        self.assertIsNone(out["next_retry_at"])
+        self.assertEqual(retry_due_automations(self.conn, "p1",
+                                               now=T0 + 61), [])
+
+    def test_retry_due_boundary(self):
+        auto = self._auto()
+        record_automation_outcome(self.conn, auto["id"], "failed",
+                                  now=T0)
+        self.assertEqual(retry_due_automations(self.conn, "p1",
+                                               now=T0 + 59), [])
+        due = retry_due_automations(self.conn, "p1", now=T0 + 60)
+        self.assertEqual([d["id"] for d in due], [auto["id"]])
+
+    def test_claim_retry_single_winner_and_clock_untouched(self):
+        auto = self._auto()
+        record_automation_outcome(self.conn, auto["id"], "failed",
+                                  now=T0)
+        before = get_automation(self.conn, auto["id"])
+        self.assertTrue(claim_retry_automation(self.conn, auto["id"],
+                                               now=T0 + 60))
+        self.assertFalse(claim_retry_automation(self.conn, auto["id"],
+                                                now=T0 + 60))
+        after = get_automation(self.conn, auto["id"])
+        self.assertEqual(after["last_run_at"],
+                         before["last_run_at"])
+        self.assertIsNone(after["next_retry_at"])
+
+    def test_run_retry_automation_builds_task(self):
+        auto = self._auto()
+        record_automation_outcome(self.conn, auto["id"], "failed",
+                                  now=T0)
+        run = run_retry_automation(self.conn, auto["id"], now=T0 + 60)
+        self.assertIsNotNone(run)
+        self.assertEqual(run["status"], "created")
+        self.assertIsNotNone(run["task_id"])
+        self.assertIsNone(run_retry_automation(self.conn, auto["id"],
+                                               now=T0 + 60))
+
+    def test_run_retry_before_due_returns_none(self):
+        auto = self._auto()
+        record_automation_outcome(self.conn, auto["id"], "failed",
+                                  now=T0)
+        self.assertIsNone(run_retry_automation(self.conn, auto["id"],
+                                               now=T0 + 59))
+
+    def test_tick_caps_at_five_interval_first(self):
+        for i in range(4):
+            self._auto("due%d" % i, first_run_at=T0 - 10)
+        retry_ids = []
+        for i in range(2):
+            auto = self._auto("retry%d" % i,
+                              first_run_at=T0 + 100000)
+            record_automation_outcome(self.conn, auto["id"], "failed",
+                                      now=T0)
+            retry_ids.append(auto["id"])
+        tick = run_cron_tick(self.conn, "p1", now=T0 + 61)
+        self.assertEqual(len(tick["due"]), 4)
+        self.assertEqual(len(tick["retry_due"]), 1)
+        self.assertTrue(tick["capped"])
+        # 两重试任务 next_retry_at 与 created_at 完全相同，
+        # 排序按 id 排，不按创建顺序；只断言属于其中之一。
+        self.assertIn(tick["retry_due"][0]["id"], retry_ids)
+
+    def test_tick_empty_when_nothing_due(self):
+        self._auto(first_run_at=T0 + 100000)
+        tick = run_cron_tick(self.conn, "p1", now=T0)
+        self.assertEqual(tick["due"], [])
+        self.assertEqual(tick["retry_due"], [])
+        self.assertFalse(tick["capped"])
+
+    def test_definition_cap_100(self):
+        for i in range(100):
+            self._auto("cap%d" % i)
+        with self.assertRaises(ValueError):
+            self._auto("cap100")
+
+    def test_update_retry_params(self):
+        auto = self._auto()
+        ok, updated = update_automation(
+            self.conn, auto["id"],
+            {"max_retries": 5, "retry_backoff_seconds": 120},
+            now=T0)
+        self.assertTrue(ok)
+        self.assertEqual(updated["max_retries"], 5)
+        self.assertEqual(updated["retry_backoff_seconds"], 120)
+        with self.assertRaises(ValueError):
+            update_automation(self.conn, auto["id"],
+                              {"max_retries": 11}, now=T0)
+        with self.assertRaises(ValueError):
+            update_automation(self.conn, auto["id"],
+                              {"retry_backoff_seconds": 5}, now=T0)
+
+    def test_create_retry_params_validated(self):
+        with self.assertRaises(ValueError):
+            self._auto("bad1", max_retries=11)
+        with self.assertRaises(ValueError):
+            self._auto("bad2", retry_backoff_seconds=4000)
+        auto = self._auto("ok", max_retries=0,
+                          retry_backoff_seconds=10)
+        self.assertEqual(auto["max_retries"], 0)
+
+
+class SwarmTestCase(unittest.TestCase):
+    def setUp(self):
+        self.tmp = tempfile.TemporaryDirectory()
+        self.state_dir = os.path.join(self.tmp.name, "state")
+        os.makedirs(self.state_dir)
+        self.root = os.path.join(self.tmp.name, "workspaces")
+        os.makedirs(self.root)
+
+    def tearDown(self):
+        self.tmp.cleanup()
+
+    def _swarm(self, max_agents=8):
+        return create_swarm(self.state_dir, "p1", "parent-1",
+                            max_agents=max_agents, now=T0)
+
+    def test_create_swarm_defaults_and_caps(self):
+        swarm = self._swarm()
+        self.assertEqual(swarm["max_agents"], 8)
+        self.assertEqual(swarm["status"], "active")
+        with self.assertRaises(ValueError):
+            create_swarm(self.state_dir, "p1", "parent-1",
+                         max_agents=0, now=T0)
+        with self.assertRaises(ValueError):
+            create_swarm(self.state_dir, "p1", "parent-1",
+                         max_agents=33, now=T0)
+
+    def test_project_swarm_cap_10(self):
+        for _ in range(10):
+            self._swarm()
+        with self.assertRaises(ValueError):
+            self._swarm()
+
+    def test_spawn_agent_own_workspace(self):
+        swarm = self._swarm()
+        a1 = spawn_agent(self.state_dir, swarm["id"], "task one",
+                         self.root, now=T0)
+        a2 = spawn_agent(self.state_dir, swarm["id"], "task two",
+                         self.root, now=T0)
+        self.assertNotEqual(a1["task_id"], a2["task_id"])
+        self.assertNotEqual(a1["workspace"], a2["workspace"])
+        self.assertTrue(a1["workspace"].startswith(self.root))
+        self.assertEqual(a1["status"], "pending")
+
+    def test_spawn_beyond_max_agents_fails(self):
+        swarm = self._swarm(max_agents=1)
+        spawn_agent(self.state_dir, swarm["id"], "one", self.root,
+                    now=T0)
+        with self.assertRaises(ValueError):
+            spawn_agent(self.state_dir, swarm["id"], "two", self.root,
+                        now=T0)
+
+    def test_check_agent_path_isolation(self):
+        swarm = self._swarm()
+        a1 = spawn_agent(self.state_dir, swarm["id"], "one",
+                         self.root, now=T0)
+        a2 = spawn_agent(self.state_dir, swarm["id"], "two",
+                         self.root, now=T0)
+        self.assertTrue(check_agent_path(
+            self.state_dir, swarm["id"], a1["id"],
+            os.path.join(a1["workspace"], "out.txt")))
+        self.assertTrue(check_agent_path(
+            self.state_dir, swarm["id"], a1["id"], a1["workspace"]))
+        self.assertFalse(check_agent_path(
+            self.state_dir, swarm["id"], a1["id"], a2["workspace"]))
+        self.assertFalse(check_agent_path(
+            self.state_dir, swarm["id"], a1["id"], self.root))
+        self.assertFalse(check_agent_path(
+            self.state_dir, swarm["id"], a1["id"],
+            os.path.join(a1["workspace"], "..", "escape.txt")))
+        self.assertFalse(check_agent_path(
+            self.state_dir, swarm["id"], "nope", a1["workspace"]))
+
+    def test_swarm_status_derived_from_agents(self):
+        swarm = self._swarm()
+        a1 = spawn_agent(self.state_dir, swarm["id"], "one",
+                         self.root, now=T0)
+        a2 = spawn_agent(self.state_dir, swarm["id"], "two",
+                         self.root, now=T0)
+        set_agent_status(self.state_dir, swarm["id"], a1["id"],
+                         "done", now=T0)
+        self.assertEqual(get_swarm(self.state_dir, swarm["id"])
+                         ["status"], "active")
+        set_agent_status(self.state_dir, swarm["id"], a2["id"],
+                         "failed", now=T0)
+        self.assertEqual(get_swarm(self.state_dir, swarm["id"])
+                         ["status"], "failed")
+        with self.assertRaises(ValueError):
+            spawn_agent(self.state_dir, swarm["id"], "three",
+                        self.root, now=T0)
+
+    def test_set_agent_status_invalid(self):
+        swarm = self._swarm()
+        agent = spawn_agent(self.state_dir, swarm["id"], "one",
+                            self.root, now=T0)
+        with self.assertRaises(ValueError):
+            set_agent_status(self.state_dir, swarm["id"], agent["id"],
+                             "nope", now=T0)
+        with self.assertRaises(ValueError):
+            set_agent_status(self.state_dir, swarm["id"], "nope",
+                             "done", now=T0)
+
+    def test_list_agents_sorted(self):
+        swarm = self._swarm()
+        spawn_agent(self.state_dir, swarm["id"], "one", self.root,
+                    now=T0)
+        spawn_agent(self.state_dir, swarm["id"], "two", self.root,
+                    now=T0 + 1)
+        agents = list_agents(self.state_dir, swarm["id"])
+        self.assertEqual(len(agents), 2)
+        self.assertEqual(agents[0]["prompt"], "one")
+        with self.assertRaises(ValueError):
+            list_agents(self.state_dir, "nope")
+
+    def test_register_tool_namespaced_per_agent(self):
+        swarm = self._swarm()
+        a1 = spawn_agent(self.state_dir, swarm["id"], "one",
+                         self.root, now=T0)
+        a2 = spawn_agent(self.state_dir, swarm["id"], "two",
+                         self.root, now=T0)
+        tool = register_tool(self.state_dir, swarm["id"], a1["id"],
+                             "search", "web search", now=T0)
+        self.assertEqual(tool["name"], "search")
+        with self.assertRaises(ValueError):
+            register_tool(self.state_dir, swarm["id"], a1["id"],
+                          "search", "again", now=T0)
+        register_tool(self.state_dir, swarm["id"], a2["id"],
+                      "search", "same name other agent", now=T0)
+        with self.assertRaises(ValueError):
+            register_tool(self.state_dir, swarm["id"], a1["id"],
+                          "x" * 101, "too long", now=T0)
+
+    def test_record_tool_invocation_scoped(self):
+        swarm = self._swarm()
+        a1 = spawn_agent(self.state_dir, swarm["id"], "one",
+                         self.root, now=T0)
+        a2 = spawn_agent(self.state_dir, swarm["id"], "two",
+                         self.root, now=T0)
+        register_tool(self.state_dir, swarm["id"], a1["id"],
+                      "search", "web search", now=T0)
+        record = record_tool_invocation(
+            self.state_dir, swarm["id"], a1["id"], "search",
+            "q=example", now=T0)
+        self.assertEqual(record["name"], "search")
+        with self.assertRaises(ValueError):
+            record_tool_invocation(self.state_dir, swarm["id"],
+                                   a2["id"], "search", "q=x", now=T0)
+        with self.assertRaises(ValueError):
+            record_tool_invocation(self.state_dir, swarm["id"],
+                                   a1["id"], "nosuch", "q=x", now=T0)
+
+    def test_dispatch_agent_runs_runner_once(self):
+        swarm = self._swarm()
+        agent = spawn_agent(self.state_dir, swarm["id"], "do it",
+                            self.root, now=T0)
+        seen = []
+
+        def runner(task):
+            seen.append(task)
+            return {"status": "ok", "task_id": task["id"]}
+
+        result = dispatch_agent(self.state_dir, swarm["id"],
+                                agent["id"], runner, now=T0)
+        self.assertEqual(result["status"], "ok")
+        self.assertEqual(seen[0]["id"], agent["task_id"])
+        self.assertEqual(seen[0]["workspace"], agent["workspace"])
+        with self.assertRaises(ValueError):
+            dispatch_agent(self.state_dir, swarm["id"], agent["id"],
+                           runner, now=T0)
+
+
+if __name__ == "__main__":
+    unittest.main()
+
+
+class PresentationTestCase(unittest.TestCase):
+    def setUp(self):
+        self.tmp = tempfile.TemporaryDirectory()
+        self.state_dir = os.path.join(self.tmp.name, "state")
+        os.makedirs(self.state_dir)
+        self.ledger = NotificationLedger(self.state_dir)
+
+    def tearDown(self):
+        self.tmp.cleanup()
+
+    def test_needs_input_and_failure_present(self):
+        self.assertEqual(
+            evaluate_presentation({"status": "needs_input"}),
+            {"present": True, "reason": "needs_input"})
+        self.assertEqual(
+            evaluate_presentation({"status": "failed"}),
+            {"present": True, "reason": "failed_needs_decision"})
+        self.assertEqual(
+            evaluate_presentation({"status": "interrupted"}),
+            {"present": True, "reason": "failed_needs_decision"})
+
+    def test_completed_with_new_content_presents(self):
+        self.assertTrue(evaluate_presentation(
+            {"status": "completed",
+             "new_artifacts": ["report.pdf"]})["present"])
+        self.assertTrue(evaluate_presentation(
+            {"status": "completed",
+             "changed_files": ["a.py"]})["present"])
+        self.assertTrue(evaluate_presentation(
+            {"status": "completed", "notify": True})["present"])
+
+    def test_routine_completed_and_cancelled_log_only(self):
+        self.assertEqual(
+            evaluate_presentation({"status": "completed"}),
+            {"present": False, "reason": "routine_completed"})
+        self.assertEqual(
+            evaluate_presentation({"status": "completed",
+                                   "new_artifacts": [],
+                                   "changed_files": []}),
+            {"present": False, "reason": "routine_completed"})
+        self.assertEqual(
+            evaluate_presentation({"status": "cancelled"}),
+            {"present": False, "reason": "cancelled"})
+
+    def test_unknown_status_fail_closed(self):
+        self.assertEqual(
+            evaluate_presentation({"status": "weird"}),
+            {"present": False, "reason": "unknown_status"})
+        self.assertEqual(
+            evaluate_presentation("not-a-dict"),
+            {"present": False, "reason": "unknown_status"})
+
+    def test_ledger_dedups_same_reason(self):
+        self.assertTrue(self.ledger.should_notify("t1", "failed",
+                                                  now=T0))
+        self.assertFalse(self.ledger.should_notify("t1", "failed",
+                                                   now=T0 + 10))
+        self.assertTrue(self.ledger.should_notify("t1", "needs_input",
+                                                  now=T0 + 10))
+        self.assertTrue(self.ledger.should_notify("t2", "failed",
+                                                  now=T0 + 10))
+
+    def test_ledger_window_expires(self):
+        self.assertTrue(self.ledger.should_notify("t1", "failed",
+                                                  now=T0))
+        self.assertFalse(self.ledger.should_notify("t1", "failed",
+                                                   now=T0 + 3599))
+        self.assertTrue(self.ledger.should_notify("t1", "failed",
+                                                  now=T0 + 3600))
+
+    def test_ledger_rejects_empty_keys(self):
+        with self.assertRaises(ValueError):
+            self.ledger.should_notify("", "failed", now=T0)
+        with self.assertRaises(ValueError):
+            self.ledger.should_notify("t1", "", now=T0)
+
+    def test_ledger_corrupt_file_recovers(self):
+        self.assertTrue(self.ledger.should_notify("t1", "failed",
+                                                  now=T0))
+        with open(os.path.join(self.state_dir,
+                               "notification-ledger.json"),
+                   "w") as fh:
+            fh.write("{not json")
+        self.assertTrue(self.ledger.should_notify("t2", "failed",
+                                                  now=T0))
+
+    def test_present_or_log_combined(self):
+        out = present_or_log({"status": "failed"}, self.ledger,
+                             "t1", now=T0)
+        self.assertEqual(out, {"present": True,
+                               "reason": "failed_needs_decision",
+                               "notified": True})
+        out = present_or_log({"status": "failed"}, self.ledger,
+                             "t1", now=T0 + 5)
+        self.assertTrue(out["present"])
+        self.assertFalse(out["notified"])
+        out = present_or_log({"status": "completed"}, self.ledger,
+                             "t2", now=T0)
+        self.assertEqual(out, {"present": False,
+                               "reason": "routine_completed",
+                               "notified": False})
+
+
+class FakeTransport(BrowserTransport):
+    """测试用 transport：记录 spawn/close，可注入失败。"""
+
+    def __init__(self):
+        self.spawned = []
+        self.closed = []
+        self.fail_spawn = False
+        self.fail_close = False
+
+    def spawn(self, instance):
+        if self.fail_spawn:
+            raise RuntimeError("spawn exploded")
+        self.spawned.append(instance["instance_id"])
+
+    def close(self, instance):
+        if self.fail_close:
+            raise RuntimeError("close exploded")
+        self.closed.append(instance["instance_id"])
+
+    def ping(self, instance):
+        return instance["instance_id"] in self.spawned \
+            and instance["instance_id"] not in self.closed
+
+
+class BrowserTasksTestCase(unittest.TestCase):
+    def setUp(self):
+        self.tmp = tempfile.TemporaryDirectory()
+        self.state_dir = os.path.join(self.tmp.name, "state")
+        os.makedirs(self.state_dir)
+        self.transport = FakeTransport()
+
+    def tearDown(self):
+        self.tmp.cleanup()
+
+    def test_create_and_get_instance(self):
+        inst = create_instance(self.state_dir, "task-1",
+                               self.transport, now=T0)
+        self.assertEqual(inst["status"], "active")
+        self.assertEqual(inst["task_id"], "task-1")
+        self.assertEqual(self.transport.spawned, [inst["instance_id"]])
+        got = get_instance(self.state_dir, "task-1")
+        self.assertEqual(got["instance_id"], inst["instance_id"])
+        self.assertIsNone(get_instance(self.state_dir, "nope"))
+
+    def test_two_tasks_get_distinct_instances(self):
+        a = create_instance(self.state_dir, "task-1", self.transport,
+                            now=T0)
+        b = create_instance(self.state_dir, "task-2", self.transport,
+                            now=T0)
+        self.assertNotEqual(a["instance_id"], b["instance_id"])
+        self.assertNotEqual(
+            instance_shots_dir(self.state_dir, "task-1"),
+            instance_shots_dir(self.state_dir, "task-2"))
+
+    def test_second_create_same_task_fails(self):
+        create_instance(self.state_dir, "task-1", self.transport,
+                        now=T0)
+        with self.assertRaises(ValueError):
+            create_instance(self.state_dir, "task-1", self.transport,
+                            now=T0)
+
+    def test_destroy_marks_destroyed_and_rejects_reuse(self):
+        inst = create_instance(self.state_dir, "task-1",
+                               self.transport, now=T0)
+        destroyed = destroy_instance(self.state_dir, "task-1",
+                                     self.transport, now=T0 + 5)
+        self.assertEqual(destroyed["status"], "destroyed")
+        self.assertEqual(destroyed["destroyed_at"], T0 + 5)
+        self.assertEqual(self.transport.closed, [inst["instance_id"]])
+        with self.assertRaises(ValueError):
+            destroy_instance(self.state_dir, "task-1", self.transport,
+                             now=T0 + 6)
+        with self.assertRaises(ValueError):
+            create_instance(self.state_dir, "task-1", self.transport,
+                            now=T0 + 6)
+        self.assertFalse(ping_instance(self.state_dir, "task-1",
+                                      self.transport))
+
+    def test_destroy_missing_fails(self):
+        with self.assertRaises(ValueError):
+            destroy_instance(self.state_dir, "nope", self.transport,
+                             now=T0)
+
+    def test_spawn_failure_records_failed(self):
+        self.transport.fail_spawn = True
+        with self.assertRaises(RuntimeError):
+            create_instance(self.state_dir, "task-1", self.transport,
+                            now=T0)
+        got = get_instance(self.state_dir, "task-1")
+        self.assertEqual(got["status"], "failed")
+
+    def test_close_failure_still_destroys(self):
+        create_instance(self.state_dir, "task-1", self.transport,
+                        now=T0)
+        self.transport.fail_close = True
+        destroyed = destroy_instance(self.state_dir, "task-1",
+                                     self.transport, now=T0)
+        self.assertEqual(destroyed["status"], "destroyed")
+        self.assertIn("close exploded", destroyed["close_error"])
+
+    def test_ping_active_and_missing(self):
+        self.assertFalse(ping_instance(self.state_dir, "task-1",
+                                      self.transport))
+        create_instance(self.state_dir, "task-1", self.transport,
+                        now=T0)
+        self.assertTrue(ping_instance(self.state_dir, "task-1",
+                                     self.transport))
+
+    def test_shots_dir_inside_state_dir(self):
+        create_instance(self.state_dir, "task-1", self.transport,
+                        now=T0)
+        shots = instance_shots_dir(self.state_dir, "task-1")
+        self.assertTrue(shots.startswith(self.state_dir))
+        self.assertIsNone(instance_shots_dir(self.state_dir, "nope"))
+
+    def test_base_transport_raises(self):
+        base = BrowserTransport()
+        with self.assertRaises(NotImplementedError):
+            base.spawn({})
+        with self.assertRaises(NotImplementedError):
+            base.close({})
+        with self.assertRaises(NotImplementedError):
+            base.ping({})