codex-app-server/cloud/tests/test_v2cov_server_handlers3.py

"""Coverage batch v2 (D3): cloud/server.py handler validation branches,
second sweep.

Same stubbing discipline as D2: the method under test is never
stubbed; _send_json / _read_json / _read_body are captured input,
_db is a delegating proxy only for the TOCTOU / failure-injection
cases, and collaborator functions (request_kill, _auto_process_due,
_pushback_block, write_control, read_state, start_oauth_login,
build_authorization_url, handle_oauth_callback, run_plan,
auto_config_plan, _validate_sub2api_key, _rotate_sub2api_key,
WebhookDeliveryStore) are stubbed only to drive the handler's own
error mapping. All validation, permission, quota and state logic
runs for real.
"""

import hashlib
import hmac
import json
import os
import sqlite3
import sys
import tempfile
import time
import unittest
from pathlib import Path
from types import SimpleNamespace
from unittest import mock

REPO_ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(REPO_ROOT))

from cloud.server import (  # noqa: E402
    CloudControlPlaneHandler,
    _auth_store,
    init_db,
)
from cloud.sub2api import Sub2apiError as _Sub2apiError  # noqa: E402


def _handler(db_path, upload_dir=None):
    h = CloudControlPlaneHandler.__new__(CloudControlPlaneHandler)
    h.server = SimpleNamespace(db_path=db_path, upload_dir=upload_dir)
    sent = []
    h._send_json = lambda status, obj, extra_headers=None: sent.append(
        (status, obj))
    h.sent = sent
    return h


def _conn(db_path):
    c = sqlite3.connect(db_path)
    c.row_factory = sqlite3.Row
    return c


class _RowcountCursor:
    def __init__(self, rowcount):
        self.rowcount = rowcount

    def fetchone(self):
        return None

    def fetchall(self):
        return []


class _ProxyConn:
    """Delegate everything except the statements a test targets."""

    def __init__(self, real, fail_insert_prefix=None,
                 fail_insert_exc=None, fail_select_substr=None,
                 fail_select_exc=None, rowcount_zero_prefix=None,
                 close_exc=None):
        self._real = real
        self._fail_insert_prefix = fail_insert_prefix
        self._fail_insert_exc = fail_insert_exc
        self._fail_select_substr = fail_select_substr
        self._fail_select_exc = fail_select_exc
        self._rowcount_zero_prefix = rowcount_zero_prefix
        self._close_exc = close_exc

    def execute(self, sql, params=()):
        text = sql.strip()
        if (self._fail_insert_prefix
                and text.startswith(self._fail_insert_prefix)):
            raise self._fail_insert_exc
        if (self._fail_select_substr
                and text.startswith("SELECT")
                and self._fail_select_substr in text):
            raise self._fail_select_exc
        if (self._rowcount_zero_prefix
                and text.startswith(self._rowcount_zero_prefix)):
            return _RowcountCursor(0)
        return self._real.execute(sql, params)

    def close(self):
        if self._close_exc is not None:
            raise self._close_exc
        return self._real.close()

    def __getattr__(self, name):
        return getattr(self._real, name)


class _Base(unittest.TestCase):
    def setUp(self):
        self.tmp = tempfile.TemporaryDirectory()
        self.db_path = os.path.join(self.tmp.name, "test.db")
        init_db(self.db_path)
        self.upload_dir = os.path.join(self.tmp.name, "uploads")
        os.makedirs(self.upload_dir, exist_ok=True)

    def tearDown(self):
        self.tmp.cleanup()

    def _h(self, body=None, err=None):
        h = _handler(self.db_path, upload_dir=self.upload_dir)
        h._read_json = lambda: (body, err)
        return h

    def _project(self, pid="p1", owner="u1", repo_url=None):
        c = _conn(self.db_path)
        c.execute(
            "INSERT INTO projects(id, name, owner_id, created_at,"
            " repo_url, auto_pr) VALUES(?, ?, ?, ?, ?, 0)",
            (pid, "P", owner, time.time(), repo_url))
        c.commit()
        c.close()
        with _auth_store(self.db_path) as store:
            store.create_project(pid, owner)

    def _task(self, tid="t1", pid="p1", status="completed",
              created_by="u1", **extra):
        cols = ["id", "project_id", "prompt", "status", "created_by",
                "created_at", "updated_at"]
        vals = [tid, pid, "do work", status, created_by, time.time(),
                time.time()]
        for key, val in extra.items():
            cols.append(key)
            vals.append(val)
        c = _conn(self.db_path)
        c.execute(
            "INSERT INTO tasks(%s) VALUES(%s)"
            % (", ".join(cols), ", ".join("?" * len(cols))), vals)
        c.commit()
        c.close()

    def _last(self, h):
        self.assertTrue(h.sent, "handler sent no response")
        return h.sent[-1]


class DispatchTransitionTestCase(_Base):
    def test_dispatch_empty_project_404(self):
        h = self._h()
        h._dispatch_project("u1", "GET", "/projects/", {})
        self.assertEqual(self._last(h)[0], 404)

    def test_dispatch_identity_supersede_wrong_method_404(self):
        h = self._h()
        h._dispatch_project("u1", "GET",
                            "/projects/p1/identity-memories/m1/supersede",
                            {})
        self.assertEqual(self._last(h)[0], 404)

    def test_dispatch_identity_delete_wrong_method_404(self):
        h = self._h()
        h._dispatch_project("u1", "GET",
                            "/projects/p1/identity-memories/m1", {})
        self.assertEqual(self._last(h)[0], 404)

    def test_dispatch_memory_supersede_wrong_method_404(self):
        h = self._h()
        h._dispatch_project("u1", "GET",
                            "/projects/p1/memory/m1/supersede", {})
        self.assertEqual(self._last(h)[0], 404)

    def test_dispatch_push_subscriptions_wrong_method_404(self):
        # POST creates and GET lists; any other method hits the
        # else branch (3934).
        h = self._h()
        h._dispatch_project("u1", "PUT",
                            "/projects/p1/push/subscriptions", {})
        self.assertEqual(self._last(h)[0], 404)

    def test_dispatch_todos_wrong_method_404(self):
        h = self._h()
        h._dispatch_project("u1", "PUT", "/projects/p1/todos/t1", {})
        self.assertEqual(self._last(h)[0], 404)

    def test_dispatch_mcp_servers_wrong_method_404(self):
        h = self._h()
        h._dispatch_project("u1", "POST", "/projects/p1/mcp-servers", {})
        self.assertEqual(self._last(h)[0], 404)

    def test_cancel_missing_404(self):
        h = self._h()
        h._cancel_task("u1", "ghost")
        self.assertEqual(self._last(h)[0], 404)

    def test_cancel_forbidden_403(self):
        self._project()
        self._task(status="running")
        h = self._h()
        h._cancel_task("u9", "t1")
        self.assertEqual(self._last(h)[0], 403)

    def test_cancel_kill_failure_swallowed_200(self):
        self._project()
        self._task(status="running")
        h = self._h()
        with mock.patch("cloud.server.request_kill",
                        side_effect=RuntimeError("kill down")):
            h._cancel_task("u1", "t1")
        code, payload = self._last(h)
        self.assertEqual(code, 200)
        self.assertEqual(payload["status"], "interrupted")

    def test_cancel_bundle_lookup_failure_swallowed_200(self):
        self._project()
        self._task(status="running")
        h = self._h()
        h._db = lambda: _ProxyConn(
            _conn(self.db_path),
            fail_select_substr="FROM task_attachments",
            fail_select_exc=RuntimeError("lookup down"))
        h._cancel_task("u1", "t1")
        self.assertEqual(self._last(h)[0], 200)

    def test_pause_missing_404(self):
        h = self._h()
        h._pause_task("u1", "ghost")
        self.assertEqual(self._last(h)[0], 404)

    def test_pause_forbidden_403(self):
        self._project()
        self._task(status="running")
        h = self._h()
        h._pause_task("u9", "t1")
        self.assertEqual(self._last(h)[0], 403)

    def test_pause_terminal_400(self):
        self._project()
        self._task(status="completed")
        h = self._h()
        h._pause_task("u1", "t1")
        code, payload = self._last(h)
        self.assertEqual(code, 400)
        self.assertIn("terminal", payload["error"])

    def test_pause_step_insert_failure_swallowed_200(self):
        self._project()
        self._task(status="running")
        h = self._h()
        h._db = lambda: _ProxyConn(
            _conn(self.db_path),
            fail_insert_prefix="INSERT INTO task_steps",
            fail_insert_exc=RuntimeError("steps down"))
        h._pause_task("u1", "t1")
        code, payload = self._last(h)
        self.assertEqual(code, 200)
        self.assertEqual(payload["status"], "interrupted")

    def test_release_missing_404(self):
        h = self._h(body={})
        h._release_task("u1", "ghost")
        self.assertEqual(self._last(h)[0], 404)

    def test_release_forbidden_403(self):
        self._project()
        self._task(status="merged")
        h = self._h(body={})
        h._audit_admin = lambda *args: None
        with mock.patch.dict(os.environ, {}, clear=False):
            os.environ.pop("CODEX_GLOBAL_ADMINS", None)
            h._release_task("u1", "t1")
        self.assertEqual(self._last(h)[0], 403)

    def test_release_read_error(self):
        self._project()
        self._task(status="merged")
        h = self._h(err="bad json")
        h._audit_admin = lambda *args: None
        with mock.patch.dict(os.environ, {"CODEX_GLOBAL_ADMINS": "u1"}):
            h._release_task("u1", "t1")
        self.assertEqual(self._last(h)[0], 400)

    def test_release_files_changed_must_be_list(self):
        self._project()
        self._task(status="merged")
        h = self._h(body={"files_changed": "x"})
        h._audit_admin = lambda *args: None
        with mock.patch.dict(os.environ, {"CODEX_GLOBAL_ADMINS": "u1"}):
            h._release_task("u1", "t1")
        code, payload = self._last(h)
        self.assertEqual(code, 400)
        self.assertIn("files_changed must be a list", payload["error"])

    def test_continue_merged_409(self):
        self._project()
        self._task(status="merged")
        h = self._h(body={"message": "again"})
        h._continue_task("u1", "t1")
        self.assertEqual(self._last(h)[0], 409)

    def test_continue_nonterminal_409(self):
        # A status outside {queued, running, merged} and outside
        # TERMINAL_STATUSES: only reachable with an unknown status
        # string in the row (defensive branch, not a normal state).
        self._project()
        self._task(status="bogus-status")
        h = self._h(body={"message": "again"})
        h._continue_task("u1", "t1")
        code, payload = self._last(h)
        self.assertEqual(code, 409)
        self.assertIn("only terminal", payload["error"])

    def test_continue_read_error(self):
        self._project()
        self._task(status="completed")
        h = self._h(err="bad json")
        h._continue_task("u1", "t1")
        self.assertEqual(self._last(h)[0], 400)

    def test_continue_quota_reject_returns(self):
        # _continue_task has two quota gates: the early
        # _reject_if_over_quota_for_task (8881) and the late
        # _reject_if_over_quota before the write (8964). The early
        # gate is stubbed to pass so the late gate's bare return
        # (8966) is the one under test.
        self._project()
        self._task(status="completed")
        h = self._h(body={"message": "again"})
        h._reject_if_over_quota_for_task = lambda *a: False
        h._quota_status = mock.Mock(side_effect=RuntimeError("down"))
        h._continue_task("u1", "t1")
        self.assertEqual(len(h.sent), 1)
        self.assertEqual(h.sent[0][0], 503)

    def test_continue_status_changed_409(self):
        # CAS rowcount 0: the task row changed under the handler,
        # so no message is persisted and the caller must refresh.
        self._project()
        self._task(status="completed")
        h = self._h(body={"message": "again"})
        h._db = lambda: _ProxyConn(
            _conn(self.db_path),
            rowcount_zero_prefix="UPDATE tasks SET")
        h._continue_task("u1", "t1")
        code, payload = self._last(h)
        self.assertEqual(code, 409)
        self.assertIn("状态已变化", payload["error"])
        c = _conn(self.db_path)
        row = c.execute(
            "SELECT status FROM tasks WHERE id='t1'").fetchone()
        c.close()
        self.assertEqual(row[0], "completed")


class TodosAutomationLoopTestCase(_Base):
    def _auto(self, name="auto1", project_id="p1", prompt="do it"):
        from cloud.automations import create_automation
        c = _conn(self.db_path)
        auto = create_automation(c, project_id, name, prompt, "u1")
        c.close()
        return auto

    def _todo(self, tid="todo1", pid="p1"):
        from cloud.todos import create_todo
        c = _conn(self.db_path)
        todo = create_todo(c, pid, "buy milk", "u1")
        c.close()
        return todo

    def test_todos_update_read_error(self):
        h = self._h(err="bad json")
        h._todos_update("u1", "p1", "todo1")
        self.assertEqual(self._last(h)[0], 400)

    def test_todos_update_value_error_400(self):
        todo = self._todo()
        h = self._h(body={"status": "bogus"})
        h._todos_update("u1", "p1", todo["id"])
        self.assertEqual(self._last(h)[0], 400)

    def test_todos_update_generic_error_502(self):
        todo = self._todo()
        h = self._h(body={"title": "new"})
        with mock.patch("cloud.handler_automation._todo_update",
                        side_effect=RuntimeError("db down")):
            h._todos_update("u1", "p1", todo["id"])
        self.assertEqual(self._last(h)[0], 502)

    def test_todos_update_missing_404(self):
        h = self._h(body={"title": "new"})
        h._todos_update("u1", "p1", "ghost")
        self.assertEqual(self._last(h)[0], 404)

    def test_todos_update_cross_project_404(self):
        todo = self._todo()
        h = self._h(body={"title": "new"})
        h._todos_update("u1", "p2", todo["id"])
        self.assertEqual(self._last(h)[0], 404)

    def test_automation_update_read_error(self):
        h = self._h(err="bad json")
        h._automation_update("u1", "p1", "a1")
        self.assertEqual(self._last(h)[0], 400)

    def test_automation_update_bad_first_run(self):
        auto = self._auto()
        h = self._h(body={"first_run_at": "not-a-time"})
        h._automation_update("u1", "p1", auto["id"])
        code, payload = self._last(h)
        self.assertEqual(code, 400)
        self.assertIn("first_run_at", payload["error"])

    def test_automation_update_good_first_run(self):
        auto = self._auto()
        h = self._h(body={"first_run_at": "2030-01-01T00:00:00Z"})
        h._automation_update("u1", "p1", auto["id"])
        self.assertEqual(self._last(h)[0], 200)

    def test_automation_update_no_fields_400(self):
        auto = self._auto()
        h = self._h(body={})
        h._automation_update("u1", "p1", auto["id"])
        self.assertEqual(self._last(h)[0], 400)

    def test_audit_loop_get_missing_404(self):
        h = self._h()
        h._audit_loop_get("u1", "ghost", {})
        self.assertEqual(self._last(h)[0], 404)

    def test_audit_loop_get_forbidden_403(self):
        self._project()
        self._task()
        h = self._h()
        h._audit_loop_get("u9", "t1", {})
        self.assertEqual(self._last(h)[0], 403)

    def test_audit_loop_get_query_required_400(self):
        self._project()
        self._task()
        h = self._h()
        h._audit_loop_get("u1", "t1", {})
        self.assertEqual(self._last(h)[0], 400)

    def test_audit_loop_get_no_state_404(self):
        self._project()
        self._task()
        h = self._h()
        h._audit_loop_get("u1", "t1", {"module_id": ["m"],
                                       "contract_version": ["v1"]})
        self.assertEqual(self._last(h)[0], 404)

    def test_audit_loop_get_pushback_unavailable_503(self):
        self._project()
        self._task()
        from cloud.audit_loop_wiring import (
            save_loop_state,
        )
        from cloud.audit_loop_driver import LoopState
        state = LoopState(module_id="m", contract_version="v1")
        c = _conn(self.db_path)
        save_loop_state(c, "t1", state)
        c.commit()
        c.close()
        h = self._h()
        with mock.patch("cloud.server._pushback_block",
                        side_effect=RuntimeError("chain down")):
            h._audit_loop_get("u1", "t1", {"module_id": ["m"],
                                           "contract_version": ["v1"]})
        code, payload = self._last(h)
        self.assertEqual(code, 503)
        self.assertEqual(payload["error"],
                         "audit_pushback_state_unavailable")

    def test_audit_loop_get_success_200(self):
        self._project()
        self._task()
        from cloud.audit_loop_wiring import save_loop_state
        from cloud.audit_loop_driver import LoopState
        state = LoopState(module_id="m", contract_version="v1")
        c = _conn(self.db_path)
        save_loop_state(c, "t1", state)
        c.commit()
        c.close()
        h = self._h()
        h._audit_loop_get("u1", "t1", {"module_id": ["m"],
                                       "contract_version": ["v1"]})
        code, payload = self._last(h)
        self.assertEqual(code, 200)
        self.assertEqual(payload["loop_state"]["module_id"], "m")

    def test_auto_memory_run_read_error(self):
        h = self._h(err="bad json")
        h._auto_memory_run("u1")
        self.assertEqual(self._last(h)[0], 400)

    def test_auto_memory_run_bad_force(self):
        h = self._h(body={"force": "yes"})
        h._auto_memory_run("u1")
        code, payload = self._last(h)
        self.assertEqual(code, 400)
        self.assertIn("force must be a boolean", payload["error"])

    def test_auto_memory_run_extracted_audits(self):
        h = self._h(body={})
        with mock.patch("cloud.handler_identity._auto_process_due",
                        return_value=[{"status": "extracted",
                                       "task_id": "t1"},
                                      {"status": "skipped",
                                       "task_id": "t2"}]):
            h._auto_memory_run("u1")
        code, payload = self._last(h)
        self.assertEqual(code, 200)
        self.assertEqual(len(payload["results"]), 2)

    def test_kb_index_read_error(self):
        h = self._h(err="bad json")
        h._kb_index("u1", "p1")
        self.assertEqual(self._last(h)[0], 400)

    def test_kb_index_content_required(self):
        h = self._h(body={"path": "a.txt"})
        h._kb_index("u1", "p1")
        code, payload = self._last(h)
        self.assertEqual(code, 400)
        self.assertIn("content required", payload["error"])

    def test_kb_index_bad_kind_400(self):
        # KBStore.index_file rejects an unknown kind with ValueError;
        # path traversal alone is not rejected by the store.
        h = self._h(body={"path": "a.txt", "content": "x",
                          "kind": "bogus"})
        h._kb_index("u1", "p1")
        code, payload = self._last(h)
        self.assertEqual(code, 400)
        self.assertIn("kind must be", payload["error"])

    def test_kb_index_success_201(self):
        self._project()
        h = self._h(body={"path": "docs/a.txt", "content": "hello"})
        h._kb_index("u1", "p1")
        code, payload = self._last(h)
        self.assertEqual(code, 201)
        self.assertEqual(payload["path"], "docs/a.txt")


class ThreadDeliverTestCase(_Base):
    def _read_h(self, task_status="running", **task_extra):
        self._project()
        self._task(status=task_status, **task_extra)
        h = self._h()
        h._trace_snapshot = lambda task: ([], None)
        return h

    def test_thread_read_task_missing(self):
        h = self._h()
        h._task_thread_read("u1", "ghost")
        code, payload = self._last(h)
        self.assertEqual(code, 404)

    def test_thread_read_heartbeat_age(self):
        h = self._read_h(last_heartbeat=time.time() - 30)
        h._trace_snapshot = lambda task: (
            [{"kind": "message", "text": "hello", "id": "e1"}], None)
        h._thread_task_guard = lambda task: ("g", "guarded")
        h._task_thread_read("u1", "t1")
        code, payload = self._last(h)
        self.assertEqual(code, 200)
        self.assertIsNotNone(payload["progress"]["heartbeat_age_seconds"])
        self.assertEqual(payload["progress"]["last_trace"]["kind"],
                         "message")
        self.assertEqual(payload["note"], "guarded")

    def test_thread_read_resolve_error_note(self):
        h = self._read_h()
        h._thread_task_guard = lambda task: None
        h._resolve_task_thread = lambda task: (
            None, None, None, (502, "resolve failed"), None)
        h._task_thread_read("u1", "t1")
        code, payload = self._last(h)
        self.assertEqual(code, 200)
        self.assertEqual(payload["note"], "resolve failed")

    def test_thread_read_agent_chain_failure_swallowed(self):
        h = self._read_h()
        h._thread_task_guard = lambda task: None
        h._resolve_task_thread = lambda task: (
            {"id": "th1"}, "/tmp", "/tmp", None, None)
        with mock.patch("cloud.server._run_host_cli",
                        return_value=(False, "no host")), \
                mock.patch("cloud.server._build_agent_chain",
                           side_effect=RuntimeError("chain down")):
            h._task_thread_read("u1", "t1")
        code, payload = self._last(h)
        self.assertEqual(code, 200)
        self.assertEqual(payload["agents"], [])

    def test_thread_read_attention_reasons(self):
        h = self._read_h(task_status="completed", error_text="boom")
        h._thread_task_guard = lambda task: ("g", "guarded")
        from cloud.server import _appr_ensure, _peer_ensure_followups
        c = _conn(self.db_path)
        _appr_ensure(c)
        c.execute(
            "INSERT INTO task_approvals(id, task_id, project_id, kind,"
            " summary, status, created_at) VALUES(?, ?, ?, ?, ?, ?, ?)",
            ("ap1", "t1", "p1", "deploy", "ok?", "pending",
             time.time()))
        _peer_ensure_followups(c)
        c.execute(
            "INSERT INTO task_followups(id, task_id, project_id, body,"
            " mode, status, created_by, created_at, updated_at)"
            " VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?)",
            ("fu1", "t1", "p1", "later", "queued", "pending", "u1",
             time.time(), time.time()))
        c.commit()
        c.close()
        h._task_thread_read("u1", "t1")
        code, payload = self._last(h)
        self.assertEqual(code, 200)
        reasons = payload["attention"]["reasons"]
        self.assertIn("pending approvals need a decision", reasons)
        self.assertIn("pending followups need processing", reasons)
        self.assertIn("terminal task can be continued"
                      " with a new instruction", reasons)
        self.assertIn("task has an error_text", reasons)

    def test_thread_message_read_error(self):
        h = self._h(err="bad json")
        h._task_thread_message("u1", "t1")
        self.assertEqual(self._last(h)[0], 400)

    def test_thread_message_task_missing(self):
        h = self._h(body={"body": "hi"})
        h._task_thread_message("u1", "ghost")
        self.assertEqual(self._last(h)[0], 404)

    def test_thread_message_quota_reject_returns(self):
        self._project()
        self._task(status="running")
        h = self._h(body={"body": "hi"})
        h._quota_status = mock.Mock(side_effect=RuntimeError("down"))
        h._task_thread_message("u1", "t1")
        self.assertEqual(len(h.sent), 1)
        self.assertEqual(h.sent[0][0], 503)

    def test_thread_message_body_required(self):
        self._project()
        self._task(status="running")
        h = self._h(body={"body": "   "})
        h._task_thread_message("u1", "t1")
        code, payload = self._last(h)
        self.assertEqual(code, 400)
        self.assertIn("body required", payload["error"])

    def test_thread_message_body_too_long(self):
        self._project()
        self._task(status="running")
        h = self._h(body={"body": "x" * 20001})
        h._task_thread_message("u1", "t1")
        code, payload = self._last(h)
        self.assertEqual(code, 400)
        self.assertIn("body too long", payload["error"])

    def _deliver_task(self, status="running", **extra):
        return {"id": "t1", "project_id": "p1", "status": status,
                "created_by": "u1", **extra}

    def test_deliver_goal_error(self):
        self._project()
        h = self._h()
        code, payload = h._deliver_to_thread(
            "u1", self._deliver_task(), "/goal", "queued")
        self.assertEqual(code, 400)

    def test_deliver_goal_convert_queued_success(self):
        self._project()
        self._task(status="queued")
        h = self._h()
        code, payload = h._deliver_to_thread(
            "u1", self._deliver_task(status="queued"),
            "/goal 做目标", "queued")
        self.assertEqual(code, 201)
        self.assertEqual(payload["delivery"], "goal_converted")

    def test_deliver_goal_convert_queued_race_409(self):
        self._project()
        h = self._h()
        h._goal_convert_queued = lambda tid, target: None
        code, payload = h._deliver_to_thread(
            "u1", self._deliver_task(status="queued"),
            "/goal 做目标", "queued")
        self.assertEqual(code, 409)
        self.assertIn("状态已变化", payload["error"])

    def test_deliver_goal_revive_race_409(self):
        self._project()
        h = self._h()
        h._goal_revive_terminal = lambda tid, target: None
        code, payload = h._deliver_to_thread(
            "u1", self._deliver_task(status="completed"),
            "/goal 做目标", "queued")
        self.assertEqual(code, 409)
        self.assertIn("pending resume message", payload["error"])

    def test_deliver_terminal_cas_race_409(self):
        self._project()
        self._task(status="completed", resume_message="old")
        h = self._h()
        code, payload = h._deliver_to_thread(
            "u1", self._deliver_task(status="completed",
                                     resume_message="old"),
            "fix it", "queued")
        self.assertEqual(code, 409)
        self.assertIn("pending resume message", payload["error"])

    def test_deliver_running_goal_queued(self):
        self._project()
        self._task(status="running")
        h = self._h()
        code, payload = h._deliver_to_thread(
            "u1", self._deliver_task(status="running"),
            "/goal 做目标", "queued")
        self.assertEqual(code, 201)
        self.assertEqual(payload["delivery"], "goal_queued")
        self.assertEqual(payload["goal"], "做目标")


class OauthGatewayTestCase(_Base):
    def test_oauth_login_unknown_provider_404(self):
        # _oauth_provider returns None only for a malformed path
        # (not /oauth/{provider}/login); any segment in the provider
        # slot is passed through to the login flow.
        h = self._h()
        h.path = "/login"
        h._oauth_login("/login")
        self.assertEqual(self._last(h)[0], 404)

    def test_oauth_login_missing_params_400(self):
        h = self._h()
        h.path = "/oauth/github/login"
        h._oauth_login("/oauth/github/login")
        code, payload = self._last(h)
        self.assertEqual(code, 400)
        self.assertIn("client_id and redirect_uri", payload["error"])

    def test_oauth_login_oauth_error_400(self):
        from cloud.oauth import OAuthError
        h = self._h()
        h.path = ("/oauth/github/login?client_id=c"
                  "&redirect_uri=https://example.com/cb")
        err = OAuthError("denied", reason="bad_client")
        with mock.patch("cloud.handler_oauth.start_oauth_login",
                        side_effect=err):
            h._oauth_login("/oauth/github/login")
        code, payload = self._last(h)
        self.assertEqual(code, 400)
        self.assertEqual(payload["reason"], "bad_client")

    def test_oauth_login_value_error_400(self):
        h = self._h()
        h.path = ("/oauth/github/login?client_id=c"
                  "&redirect_uri=https://example.com/cb")
        with mock.patch("cloud.handler_oauth.start_oauth_login",
                        side_effect=ValueError("bad scope")):
            h._oauth_login("/oauth/github/login")
        self.assertEqual(self._last(h)[0], 400)

    def test_oauth_bind_unauthorized_401(self):
        h = self._h(body={})
        h._authenticate = lambda: None
        h._oauth_bind_initiate("github")
        self.assertEqual(self._last(h)[0], 401)

    def test_oauth_bind_oauth_error_400(self):
        from cloud.oauth import OAuthError
        h = self._h(body={"client_id": "c",
                          "redirect_uri": "https://example.com/cb"})
        h._authenticate = lambda: "u1"
        err = OAuthError("denied", reason="bad_client")
        with mock.patch("cloud.handler_oauth.build_authorization_url",
                        side_effect=err):
            h._oauth_bind_initiate("github")
        code, payload = self._last(h)
        self.assertEqual(code, 400)
        self.assertEqual(payload["reason"], "bad_client")

    def test_oauth_bind_value_error_400(self):
        h = self._h(body={"client_id": "c",
                          "redirect_uri": "https://example.com/cb"})
        h._authenticate = lambda: "u1"
        with mock.patch("cloud.handler_oauth.build_authorization_url",
                        side_effect=ValueError("bad scope")):
            h._oauth_bind_initiate("github")
        self.assertEqual(self._last(h)[0], 400)

    def test_oauth_bind_store_close_failure_swallowed(self):
        h = self._h(body={"client_id": "c",
                          "redirect_uri": "https://example.com/cb"})
        h._authenticate = lambda: "u1"

        class _Store:
            def __init__(self, db_path):
                self.conn = mock.Mock()
                self.conn.close.side_effect = RuntimeError("close down")

            def create_state(self, provider, redirect_uri, client_id=None,
                             bind_user_id=None):
                return "st1"
        with mock.patch("cloud.handler_oauth.OAuthStateStore", _Store), \
                mock.patch("cloud.handler_oauth.build_authorization_url",
                           return_value="https://example.com/auth"):
            h._oauth_bind_initiate("github")
        code, payload = self._last(h)
        self.assertEqual(code, 200)
        self.assertEqual(payload["state"], "st1")

    def _finish_h(self, identity):
        h = self._h()
        h._oauth_clients = lambda provider: (None, None)
        patcher = mock.patch("cloud.handler_oauth.handle_oauth_callback",
                             return_value=dict(identity))
        patcher.start()
        self.addCleanup(patcher.stop)
        return h

    def test_oauth_finish_bind_branch_quota(self):
        with _auth_store(self.db_path) as store:
            uid, _key = store.create_user("u9")
        h = self._finish_h({"bind_user_id": uid, "external_id": "ext1"})
        result = h._oauth_finish("github", "code", "st", "c", "r")
        self.assertEqual(result[0], uid)
        self.assertTrue(result[5])
        c = _conn(self.db_path)
        row = c.execute(
            "SELECT quota_usd_micros FROM user_quotas WHERE user_id=?",
            (uid,)).fetchone()
        c.close()
        self.assertIsNotNone(row)

    def test_oauth_finish_bound_branch_quota(self):
        with _auth_store(self.db_path) as store:
            uid, _key = store.create_user("u9")
        from cloud.oauth import bind_external_identity
        with _auth_store(self.db_path) as auth:
            bind_external_identity(auth, "github",
                                   {"external_id": "ext1"}, uid)
        h = self._finish_h({"external_id": "ext1"})
        result = h._oauth_finish("github", "code", "st", "c", "r")
        self.assertEqual(result[0], uid)
        self.assertFalse(result[1])
        self.assertIsNotNone(result[2])

    def test_oauth_finish_new_user_gateway_provision_fails(self):
        # Best-effort gateway provisioning: the failure is audited
        # and account creation still succeeds (no network call: the
        # gateway collaborator itself raises).
        h = self._finish_h({"external_id": "ext-new"})
        with mock.patch.dict(os.environ,
                             {"SUB2API_BASE_URL": "https://gw.example"}), \
                mock.patch("cloud.handler_oauth._ensure_sub2api_key",
                           side_effect=RuntimeError("gw down")):
            result = h._oauth_finish("github", "code", "st", "c", "r")
        self.assertTrue(result[1])
        self.assertFalse(result[5])
        c = _conn(self.db_path)
        row = c.execute(
            "SELECT action FROM audit_log"
            " WHERE action='sub2api_provision_failed'").fetchone()
        c.close()
        self.assertIsNotNone(row)

    def test_oauth_finish_state_store_close_failure_swallowed(self):
        # The finally-block except-pass (10265-66): the OAuth state
        # store's close raises, the finish flow still proceeds.
        with _auth_store(self.db_path) as store:
            uid, _key = store.create_user("u9")
        h = self._h()
        h._oauth_clients = lambda provider: (None, None)

        class _Store:
            def __init__(self, db_path):
                self.conn = mock.Mock()
                self.conn.close.side_effect = RuntimeError("close down")
        with mock.patch("cloud.handler_oauth.OAuthStateStore", _Store), \
                mock.patch("cloud.handler_oauth.handle_oauth_callback",
                           return_value={"bind_user_id": uid,
                                         "external_id": "ext1"}):
            result = h._oauth_finish("github", "code", "st", "c", "r")
        self.assertEqual(result[0], uid)
        self.assertTrue(result[5])

    def test_gateway_rotate_unauthorized_401(self):
        h = self._h()
        h._authenticate = lambda: None
        h._user_gateway_rotate()
        self.assertEqual(self._last(h)[0], 401)

    def test_gateway_rotate_sub2api_error_502(self):
        h = self._h()
        h._authenticate = lambda: "u1"
        with mock.patch.dict(os.environ, {"CLOUD_GATEWAY": "sub2api"}), \
                mock.patch("cloud.server._rotate_sub2api_key",
                           side_effect=_Sub2apiError("gw down")):
            h._user_gateway_rotate()
        code, payload = self._last(h)
        self.assertEqual(code, 502)
        self.assertIn("gw down", payload["error"])

    def test_gateway_rotate_generic_error_502(self):
        h = self._h()
        h._authenticate = lambda: "u1"
        with mock.patch.dict(os.environ, {"CLOUD_GATEWAY": "sub2api"}), \
                mock.patch("cloud.server._rotate_sub2api_key",
                           side_effect=RuntimeError("net down")):
            h._user_gateway_rotate()
        self.assertEqual(self._last(h)[0], 502)


class WebhookBrowserFullAutoTestCase(_Base):
    def _gh_headers(self, secret, raw, event, delivery="d1"):
        sig = "sha256=" + hmac.new(
            secret.encode(), raw, hashlib.sha256).hexdigest()
        return {"X-Hub-Signature-256": sig, "X-GitHub-Event": event,
                "X-GitHub-Delivery": delivery}

    def _gh_h(self, raw, event, delivery="d1"):
        h = self._h()
        h.headers = self._gh_headers("s3cr3t", raw, event, delivery)
        h._read_body = lambda: raw
        return h

    def test_github_installation_unparsed_200(self):
        raw = json.dumps({"action": "opened",
                          "installation": {"id": 7}}).encode()
        h = self._gh_h(raw, "installation")
        with mock.patch.dict(os.environ,
                             {"GITHUB_WEBHOOK_SECRET": "s3cr3t"}):
            h._handle_github_webhook()
        code, payload = self._last(h)
        self.assertEqual(code, 200)
        self.assertIsNone(payload["result"])

    def test_github_installation_removed_200(self):
        raw = json.dumps({
            "action": "removed",
            "installation": {"id": 7, "account": {"login": "octo"}},
            "repositories_removed": [{"full_name": "o/r"}]}).encode()
        h = self._gh_h(raw, "installation")
        with mock.patch.dict(os.environ,
                             {"GITHUB_WEBHOOK_SECRET": "s3cr3t"}):
            h._handle_github_webhook()
        self.assertEqual(self._last(h)[0], 200)

    def test_github_installation_deleted_200(self):
        raw = json.dumps({
            "action": "deleted",
            "installation": {"id": 7, "account": {"login": "octo"}}}).encode()
        h = self._gh_h(raw, "installation")
        with mock.patch.dict(os.environ,
                             {"GITHUB_WEBHOOK_SECRET": "s3cr3t"}):
            h._handle_github_webhook()
        self.assertEqual(self._last(h)[0], 200)

    def test_github_delivery_store_close_failure_swallowed(self):
        raw = json.dumps({"action": "opened",
                          "installation": {"id": 7}}).encode()
        h = self._gh_h(raw, "installation")

        class _Store:
            def __init__(self, db_path):
                pass

            def claim(self, delivery_id, event=None, repo=None):
                return True

            def close(self):
                raise RuntimeError("close down")
        with mock.patch.dict(os.environ,
                             {"GITHUB_WEBHOOK_SECRET": "s3cr3t"}), \
                mock.patch("cloud.server.WebhookDeliveryStore", _Store):
            h._handle_github_webhook()
        self.assertEqual(self._last(h)[0], 200)

    def _slack_h(self, raw, content_type, secret="slack-secret"):
        h = self._h()
        ts = str(int(time.time()))
        sig = "v0=" + hmac.new(
            secret.encode(), ("v0:%s:" % ts).encode() + raw,
            hashlib.sha256).hexdigest()
        h.headers = {"X-Slack-Signature": sig,
                     "X-Slack-Request-Timestamp": ts,
                     "Content-Type": content_type}
        h._read_body = lambda: raw
        return h

    def test_slack_no_secret_503(self):
        h = self._h()
        h._read_body = lambda: b""
        h.headers = {}
        with mock.patch.dict(os.environ, {}, clear=False):
            os.environ.pop("SLACK_SIGNING_SECRET", None)
            h._handle_slack_webhook()
        self.assertEqual(self._last(h)[0], 503)

    def test_slack_form_text_200(self):
        raw = b"text=%2Ftasks+list&command=%2Ftasks"
        h = self._slack_h(raw, "application/x-www-form-urlencoded")
        with mock.patch.dict(os.environ,
                             {"SLACK_SIGNING_SECRET": "slack-secret"}):
            h._handle_slack_webhook()
        self.assertEqual(self._last(h)[0], 200)

    def test_slack_json_dict_text_200(self):
        raw = json.dumps({"text": "hello"}).encode()
        h = self._slack_h(raw, "application/json")
        with mock.patch.dict(os.environ,
                             {"SLACK_SIGNING_SECRET": "slack-secret"}):
            h._handle_slack_webhook()
        self.assertEqual(self._last(h)[0], 200)

    def test_slack_json_list_text_200(self):
        raw = b"[1, 2]"
        h = self._slack_h(raw, "application/json")
        with mock.patch.dict(os.environ,
                             {"SLACK_SIGNING_SECRET": "slack-secret"}):
            h._handle_slack_webhook()
        self.assertEqual(self._last(h)[0], 200)

    def test_slack_bad_json_text_200(self):
        raw = b"not json{{{"
        h = self._slack_h(raw, "application/json")
        with mock.patch.dict(os.environ,
                             {"SLACK_SIGNING_SECRET": "slack-secret"}):
            h._handle_slack_webhook()
        self.assertEqual(self._last(h)[0], 200)

    def test_browser_control_worktree_error(self):
        h = self._h()
        h._browser_worktree_for = lambda *a: (None, (404, "nope"))
        h._browser_control("u1", "t1", {})
        self.assertEqual(self._last(h)[0], 404)

    def test_browser_control_value_error_400(self):
        h = self._h()
        h._browser_worktree_for = lambda *a: (self.tmp.name, None)
        with mock.patch("cloud.playwright_remote.write_control",
                        side_effect=ValueError("bad holder")):
            h._browser_control("u1", "t1", {})
        self.assertEqual(self._last(h)[0], 400)

    def test_browser_control_os_error_503(self):
        h = self._h()
        h._browser_worktree_for = lambda *a: (self.tmp.name, None)
        with mock.patch("cloud.playwright_remote.write_control",
                        side_effect=OSError("unreachable")):
            h._browser_control("u1", "t1", {})
        code, payload = self._last(h)
        self.assertEqual(code, 503)
        self.assertIn("browser control unreachable", payload["error"])

    def _stream_h(self):
        h = self._h()
        h.send_response = lambda *a: None
        h.send_header = lambda *a: None
        h.end_headers = lambda: None
        h.close_connection = False
        return h

    def test_browser_stream_worktree_error(self):
        h = self._stream_h()
        h._browser_worktree_for = lambda *a: (None, (404, "nope"))
        h._browser_stream("u1", "t1")
        self.assertEqual(self._last(h)[0], 404)

    def test_browser_stream_read_error(self):
        h = self._stream_h()
        h._browser_worktree_for = lambda *a: (self.tmp.name, None)
        with mock.patch("cloud.playwright_remote.read_state",
                        return_value=(None, (502, "no session"))):
            h._browser_stream("u1", "t1")
        code, payload = self._last(h)
        self.assertEqual(code, 502)

    def test_browser_stream_task_row_gone_breaks(self):
        h = self._stream_h()
        h._browser_worktree_for = lambda *a: (self.tmp.name, None)
        h._fetch_task = lambda tid: None
        h.wfile = mock.Mock()
        # First read (pre-loop) must succeed so the loop starts;
        # the second read reports no session, then the status check
        # finds the task row gone and breaks.
        with mock.patch("cloud.playwright_remote.read_state",
                        side_effect=[({"url": "", "title": ""}, None),
                                     (None, (503, "waiting"))]):
            h._browser_stream("u1", "t1")
        self.assertTrue(h.close_connection)

    def test_browser_stream_frame_missing_broken_pipe(self):
        h = self._stream_h()
        h._browser_worktree_for = lambda *a: (self.tmp.name, None)

        class _WFile:
            def write(self, data):
                raise BrokenPipeError("client gone")

            def flush(self):
                pass
        h.wfile = _WFile()
        with mock.patch("cloud.playwright_remote.read_state",
                        return_value=({"url": "", "title": ""}, None)):
            h._browser_stream("u1", "t1")
        self.assertTrue(h.close_connection)

    def test_browser_stream_abandoned_watch_breaks(self):
        h = self._stream_h()
        h._browser_worktree_for = lambda *a: (self.tmp.name, None)
        h._fetch_task = lambda tid: {"status": "running"}
        h.wfile = mock.Mock()
        # Pre-loop read succeeds; later reads report no session
        # while the task stays alive, so staleness grows until the
        # 600s abandoned-watch break fires (clock stubbed forward).
        reads = [({"url": "", "title": ""}, None),
                 (None, (503, "waiting")),
                 (None, (503, "waiting")),
                 (None, (503, "waiting"))]
        with mock.patch("cloud.playwright_remote.read_state",
                        side_effect=reads), \
                mock.patch("time.time",
                           side_effect=[1000000.0, 1000601.0, 1001202.0]):
            h._browser_stream("u1", "t1")
        self.assertTrue(h.close_connection)

    def _full_auto_install(self, repo="o/r"):
        from cloud.github_app import (
            _ensure_installations_table,
            get_app_config,
        )
        c = _conn(self.db_path)
        get_app_config(c)
        c.execute(
            "INSERT OR REPLACE INTO github_app_config(id, app_id, slug,"
            " client_id, webhook_secret, pem, html_url, created_at)"
            " VALUES(1, '1', 'app', 'cid', 'sec', NULL, NULL, ?)",
            (time.time(),))
        _ensure_installations_table(c)
        c.execute(
            "INSERT INTO github_app_installations(installation_id,"
            " repo_full_name, created_at) VALUES(?, ?, ?)",
            (123, repo, time.time()))
        c.commit()
        c.close()

    def _full_auto_h(self, body):
        h = self._h(body)
        h._audit_admin = lambda *args: None
        return h

    def _protection_client(self):
        from cloud.full_auto import branch_protection_payload
        payload = branch_protection_payload(["verify"])
        return lambda m, u, hd, b: (200, payload)

    def test_full_auto_resolve_path_token_missing(self):
        # No test resolver: the _resolve_app_token plumbing lines
        # run (conn open/close), and a None token denies.
        self._full_auto_install()
        h = self._full_auto_h({"repo": "o/r"})
        with mock.patch.dict(os.environ, {"CODEX_GLOBAL_ADMINS": "u1",
                                          "FULL_AUTO_ENABLED": "1"}), \
                mock.patch("cloud.handler_fullauto._resolve_app_token",
                           return_value=None):
            h._full_auto_execute("u1")
        code, payload = self._last(h)
        self.assertEqual(code, 403)
        self.assertEqual(payload["reason"], "token_missing")

    def test_full_auto_plan_error_400(self):
        self._full_auto_install()
        h = self._full_auto_h({"repo": "o/r"})
        h.server.full_auto_token_resolver = lambda db_path, repo: "tok"
        h.server.full_auto_http_client = self._protection_client()
        from cloud.full_auto import FullAutoError
        with mock.patch.dict(os.environ, {"CODEX_GLOBAL_ADMINS": "u1",
                                          "FULL_AUTO_ENABLED": "1"}), \
                mock.patch("cloud.handler_fullauto.auto_config_plan",
                           side_effect=FullAutoError(
                               "bad plan", reason="bad")):
            h._full_auto_execute("u1")
        code, payload = self._last(h)
        self.assertEqual(code, 400)
        self.assertEqual(payload["reason"], "bad")

    def test_full_auto_plan_denied(self):
        self._full_auto_install()
        h = self._full_auto_h({"repo": "o/r"})
        h.server.full_auto_token_resolver = lambda db_path, repo: "tok"
        h.server.full_auto_http_client = self._protection_client()
        from cloud.full_auto import FullAutoDenied
        with mock.patch.dict(os.environ, {"CODEX_GLOBAL_ADMINS": "u1",
                                          "FULL_AUTO_ENABLED": "1"}), \
                mock.patch("cloud.handler_fullauto.auto_config_plan",
                           return_value=[{"step": "s1"}]), \
                mock.patch("cloud.handler_fullauto.run_plan",
                           side_effect=FullAutoDenied(
                               "nope", reason="step_denied")):
            h._full_auto_execute("u1")
        code, payload = self._last(h)
        self.assertEqual(code, 403)
        self.assertEqual(payload["reason"], "step_denied")

    def test_full_auto_plan_error_502(self):
        self._full_auto_install()
        h = self._full_auto_h({"repo": "o/r"})
        h.server.full_auto_token_resolver = lambda db_path, repo: "tok"
        h.server.full_auto_http_client = self._protection_client()
        from cloud.full_auto import FullAutoError
        with mock.patch.dict(os.environ, {"CODEX_GLOBAL_ADMINS": "u1",
                                          "FULL_AUTO_ENABLED": "1"}), \
                mock.patch("cloud.handler_fullauto.auto_config_plan",
                           return_value=[{"step": "s1"}]), \
                mock.patch("cloud.handler_fullauto.run_plan",
                           side_effect=FullAutoError(
                               "boom", reason="plan")):
            h._full_auto_execute("u1")
        code, payload = self._last(h)
        self.assertEqual(code, 502)
        self.assertEqual(payload["reason"], "plan")
        self.assertEqual(payload["steps"], [])

    def test_create_task_idempotency_race_no_row_raises(self):
        # The other side of the D2 race: the INSERT loses and the
        # re-read finds nothing, so the IntegrityError propagates.
        self._project()
        h = self._h(body={"project_id": "p1", "prompt": "hello",
                          "idempotency_key": "k1"})
        h._db = lambda: _ProxyConn(
            _conn(self.db_path),
            fail_insert_prefix="INSERT INTO tasks",
            fail_insert_exc=sqlite3.IntegrityError("race"))
        with self.assertRaises(sqlite3.IntegrityError):
            h._create_task("u1")