codex-app-server/cloud/tests/test_v2cov_server_handlers6.py

"""Coverage batch v2 (D6): cloud/server.py third handler cluster --
handoff/worktree/refs, sleep/wait/fork, loop helpers, approvals,
goal-resume/retry, quota/admin, metrics, oauth/github-app, full-auto,
keys, push, automation/todos, triage, dispatch routes.

Discipline as in D2-D5: the method under test is never stubbed.
Collaborators (quota gates, host-CLI runners, diag builders, store
close paths) are stubbed only to drive the handler's own mapping.
"""

import io
import os
import sqlite3
import subprocess
import sys
import tempfile
import time
import unittest
from pathlib import Path
from types import SimpleNamespace
from unittest import mock

REPO_ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(REPO_ROOT))

from cloud.server import (  # noqa: E402
    CloudControlPlaneHandler,
    _auth_store,
    init_db,
)
from cloud.full_auto import FullAutoError  # noqa: E402
from cloud.oauth import BindError  # noqa: E402
from cloud.wait_ops import create_handoff as _handoff_create  # noqa: E402


def _handler(db_path, upload_dir=None, tenants_root=None):
    h = CloudControlPlaneHandler.__new__(CloudControlPlaneHandler)
    h.server = SimpleNamespace(db_path=db_path, upload_dir=upload_dir,
                               tenants_root=tenants_root)
    sent = []
    h._send_json = lambda status, obj, extra_headers=None: sent.append(
        (status, obj))
    html = []
    h._send_html = lambda status, body: html.append((status, body))
    h.sent = sent
    h.html = html
    return h


def _conn(db_path):
    c = sqlite3.connect(db_path)
    c.row_factory = sqlite3.Row
    return c


class _NoneCursor:
    def fetchone(self):
        return None

    def fetchall(self):
        return []


class _RowcountCursor:
    def __init__(self, rowcount=0):
        self.rowcount = rowcount

    def fetchone(self):
        return None

    def fetchall(self):
        return []


class _ProxyConn:
    """Selective _db proxy: fail matching SELECTs, or rowcount-0 the
    matching UPDATEs; everything else delegates to a real connection.
    A fresh real connection is opened per _db() call by the caller.
    """

    def __init__(self, real, fail_select_substr=None, fail_exc=None,
                 fail_select_nth=None, rowcount_zero_prefix=None,
                 none_select_substr=None):
        self._real = real
        self._fail_select_substr = fail_select_substr
        self._fail_exc = fail_exc
        self._fail_select_nth = fail_select_nth
        self._select_seen = 0
        self._rowcount_zero_prefix = rowcount_zero_prefix
        self._none_select_substr = none_select_substr

    def execute(self, sql, params=()):
        text = sql.strip()
        if (self._rowcount_zero_prefix
                and text.startswith(self._rowcount_zero_prefix)):
            return _RowcountCursor(0)
        if text.startswith("SELECT"):
            if (self._none_select_substr
                    and self._none_select_substr in text):
                return _NoneCursor()
            if (self._fail_select_substr
                    and self._fail_select_substr in text):
                if self._fail_exc is not None:
                    raise self._fail_exc
            if self._fail_select_nth:
                substr, nth = self._fail_select_nth
                if substr in text:
                    self._select_seen += 1
                    if self._select_seen >= nth:
                        return _NoneCursor()
        return self._real.execute(sql, params)

    def __getattr__(self, name):
        return getattr(self._real, name)


class _FailAllConn:
    def __init__(self, real):
        self._real = real

    def execute(self, sql, params=()):
        raise RuntimeError("db down")

    def __getattr__(self, name):
        return getattr(self._real, name)


class _Base(unittest.TestCase):
    def setUp(self):
        self.tmp = tempfile.TemporaryDirectory()
        self.db_path = os.path.join(self.tmp.name, "test.db")
        init_db(self.db_path)
        self.upload_dir = os.path.join(self.tmp.name, "uploads")
        os.makedirs(self.upload_dir, exist_ok=True)

    def tearDown(self):
        self.tmp.cleanup()

    def _h(self, body=None, err=None):
        h = _handler(self.db_path, upload_dir=self.upload_dir,
                     tenants_root=self.tmp.name)
        h._read_json = lambda: (body, err)
        return h

    def _project(self, pid="p1", owner="u1", repo_url=None):
        c = _conn(self.db_path)
        c.execute(
            "INSERT INTO projects(id, name, owner_id, created_at,"
            " repo_url, auto_pr) VALUES(?, ?, ?, ?, ?, 0)",
            (pid, "P", owner, time.time(), repo_url))
        c.commit()
        c.close()
        with _auth_store(self.db_path) as store:
            store.create_project(pid, owner)

    def _member(self, pid, uid, role):
        c = _conn(self.db_path)
        c.execute(
            "INSERT OR REPLACE INTO project_members(project_id,"
            " user_id, role) VALUES(?, ?, ?)", (pid, uid, role))
        c.commit()
        c.close()

    def _task(self, tid="t1", pid="p1", status="completed",
              created_by="u1", **extra):
        cols = ["id", "project_id", "prompt", "status", "created_by",
                "created_at", "updated_at"]
        vals = [tid, pid, "do work", status, created_by, time.time(),
                time.time()]
        for key, val in extra.items():
            cols.append(key)
            vals.append(val)
        c = _conn(self.db_path)
        c.execute(
            "INSERT INTO tasks(%s) VALUES(%s)"
            % (", ".join(cols), ", ".join("?" * len(cols))), vals)
        c.commit()
        c.close()

    def _env(self, eid="e1", pid="p1", status="ready"):
        c = _conn(self.db_path)
        c.execute(
            "INSERT INTO environments(id, project_id, name, status,"
            " created_at) VALUES(?, ?, ?, ?, ?)",
            (eid, pid, "env", status, time.time()))
        c.commit()
        c.close()

    def _user_row(self, uid):
        with _auth_store(self.db_path):
            pass
        c = _conn(self.db_path)
        c.execute("INSERT OR IGNORE INTO users(id) VALUES(?)", (uid,))
        c.commit()
        c.close()

    def _last(self, h):
        self.assertTrue(h.sent, "handler sent no response")
        return h.sent[-1]


class DispatchReadTestCase(_Base):
    def test_read_body_bad_length(self):
        h = self._h()
        h.headers = {"Content-Length": "abc"}
        h.rfile = io.BytesIO(b"")
        self.assertEqual(h._read_body(), b"")

    def test_read_json_non_object(self):
        h = self._h()
        del h._read_json
        h._read_body = lambda: b"[1, 2]"
        body, err = h._read_json()
        self.assertIsNone(body)
        self.assertEqual(err, "body must be a JSON object")

    def test_dispatch_console_os_error_404(self):
        h = self._h()
        h.path = "/console"
        real_open = open

        def _open(path, *a, **k):
            if str(path).endswith("console.html"):
                raise OSError("gone")
            return real_open(path, *a, **k)
        with mock.patch("builtins.open", side_effect=_open):
            h._dispatch("GET")
        self.assertEqual(self._last(h)[0], 404)

    def test_dispatch_oauth_bind_wrong_method_404(self):
        h = self._h()
        h.headers = {}
        h.path = "/oauth/github/bindings"
        h._dispatch("PUT")
        code, payload = self._last(h)
        self.assertEqual(code, 404)
        self.assertEqual(payload["error"], "not found")

    def test_dispatch_environments_else_404(self):
        h = self._h()
        h.headers = {}
        h.client_address = ("127.0.0.1", 1234)
        h.path = "/environments/xyz/extra"
        h._authenticate = lambda: "u1"
        h._dispatch("GET")
        self.assertEqual(self._last(h)[0], 404)

    def test_dispatch_admin_cost_by_creator_route(self):
        self._project()
        self._user_row("u1")
        h = self._h()
        h.headers = {}
        h.client_address = ("127.0.0.1", 1234)
        h.path = "/admin/cost-by-creator?user_id=u1&month=2026-11"
        h._authenticate = lambda: "u1"
        with mock.patch.dict(os.environ, {"CODEX_GLOBAL_ADMINS": "u1"}):
            h._dispatch("GET")
        self.assertEqual(self._last(h)[0], 200)


class HandoffWorktreeTestCase(_Base):
    def test_handoff_create_read_error(self):
        h = self._h(err="bad json")
        h._task_handoff_create("u1", "t1")
        self.assertEqual(self._last(h)[0], 400)

    def test_handoff_create_ghost_404(self):
        h = self._h(body={})
        h._task_handoff_create("u1", "ghost")
        self.assertEqual(self._last(h)[0], 404)

    def test_handoff_get_ghost_404(self):
        h = self._h()
        h._task_handoff_get("u1", "ghost")
        self.assertEqual(self._last(h)[0], 404)

    def test_handoff_report_read_error(self):
        h = self._h(err="bad json")
        h._task_handoff_report("u1", "t1", "h1")
        self.assertEqual(self._last(h)[0], 400)

    def test_handoff_report_ghost_task_404(self):
        h = self._h(body={})
        h._task_handoff_report("u1", "ghost", "h1")
        self.assertEqual(self._last(h)[0], 404)

    def test_handoff_report_bad_status_400(self):
        self._project()
        self._task()
        c = _conn(self.db_path)
        handoff = _handoff_create(c, "t1", "p1", "host-b", "u1")
        c.commit()
        c.close()
        h = self._h(body={"status": "bogus"})
        h._task_handoff_report("u1", "t1", handoff["id"])
        code, payload = self._last(h)
        self.assertEqual(code, 400)
        self.assertIn("status must be one of", payload["error"])

    def test_handoff_report_race_not_found_404(self):
        self._project()
        self._task()
        c = _conn(self.db_path)
        handoff = _handoff_create(c, "t1", "p1", "host-b", "u1")
        c.commit()
        c.close()
        h = self._h(body={"status": "running"})
        # First task_handoffs SELECT (the get) sees the row; the
        # second (inside report_handoff) sees nothing -> the
        # TOCTOU "handoff not found" mapping.
        h._db = lambda: _ProxyConn(
            _conn(self.db_path),
            fail_select_nth=("FROM task_handoffs", 2))
        h._task_handoff_report("u1", "t1", handoff["id"])
        code, payload = self._last(h)
        self.assertEqual(code, 404)
        self.assertEqual(payload["error"], "handoff not found")

    def _git_repo(self, name="t1", commit=False):
        repo = os.path.join(self.tmp.name, "wt", name)
        os.makedirs(repo, exist_ok=True)
        subprocess.run(["git", "init", "-q", repo], check=True)
        if commit:
            with open(os.path.join(repo, "a.txt"), "w") as f:
                f.write("x")
            subprocess.run(["git", "-C", repo, "add", "."],
                           check=True)
            subprocess.run(
                ["git", "-C", repo, "-c", "user.email=a@b.c",
                 "-c", "user.name=t", "commit", "-qm", "init"],
                check=True)
        return repo

    def test_worktree_summary_empty_repo_branch_none(self):
        repo = self._git_repo(commit=True)
        subprocess.run(["git", "-C", repo, "checkout", "-q", "--detach",
                        "HEAD"], check=True)
        h = self._h()
        h.server.host_worktrees_dir = os.path.join(self.tmp.name, "wt")
        summary = h._worktree_summary({"id": "t1", "project_id": "p1",
                                       "status": "completed"})
        self.assertTrue(summary["exists"])
        self.assertTrue(summary["is_git"])
        self.assertIsNone(summary["branch"])

    def test_worktree_summary_branch_command_failure(self):
        self._git_repo()
        h = self._h()
        h.server.host_worktrees_dir = os.path.join(self.tmp.name, "wt")
        with mock.patch("cloud.server.subprocess.run",
                        side_effect=RuntimeError("git down")):
            summary = h._worktree_summary(
                {"id": "t1", "project_id": "p1", "status": "completed"})
        self.assertIsNone(summary["branch"])

    def test_worktree_summary_env_status(self):
        self._git_repo()
        self._env()
        h = self._h()
        h.server.host_worktrees_dir = os.path.join(self.tmp.name, "wt")
        summary = h._worktree_summary(
            {"id": "t1", "project_id": "p1", "status": "completed",
             "environment_id": "e1"})
        self.assertEqual(summary["environment_id"], "e1")

    def test_task_bindings_env_block(self):
        self._project()
        self._task(environment_id="e1")
        self._env()
        h = self._h()
        h._task_bindings("u1", "t1")
        self.assertEqual(self._last(h)[0], 200)

    def test_refs_list_ghost(self):
        h = self._h()
        h._task_refs_list("u1", "ghost")
        self.assertEqual(self._last(h)[0], 404)

    def test_refs_create_read_error(self):
        self._project()
        self._task()
        h = self._h(err="bad json")
        h._task_refs_create("u1", "t1")
        self.assertEqual(self._last(h)[0], 400)

    def test_refs_create_ghost(self):
        h = self._h(body={})
        h._task_refs_create("u1", "ghost")
        self.assertEqual(self._last(h)[0], 404)

    def test_refs_delete_ghost(self):
        h = self._h()
        h._task_refs_delete("u1", "ghost", "r1")
        self.assertEqual(self._last(h)[0], 404)

    def test_git_text_success_slice(self):
        repo = self._git_repo(commit=True)
        h = self._h()
        out = h._git_text(repo, ["rev-parse", "HEAD"], 7)
        self.assertEqual(len(out), 7)

    def test_git_text_command_failure_empty(self):
        h = self._h()
        with mock.patch("cloud.server.subprocess.run",
                        side_effect=RuntimeError("git down")):
            self.assertEqual(h._git_text("/tmp", ["status"], 100), "")

    def test_worktree_archive_ghost(self):
        h = self._h()
        h._task_worktree_archive("u1", "ghost")
        self.assertEqual(self._last(h)[0], 404)

    def test_worktree_archive_remote_409(self):
        self._project()
        self._task(worktree_host="other-host")
        h = self._h()
        h._task_worktree_archive("u1", "t1")
        code, payload = self._last(h)
        self.assertEqual(code, 409)
        self.assertIn("other-host", payload["error"])

    def test_worktree_snapshots_ghost(self):
        h = self._h()
        h._task_worktree_snapshots("u1", "ghost")
        self.assertEqual(self._last(h)[0], 404)

    def test_worktree_restore_ghost(self):
        h = self._h()
        h._task_worktree_restore("u1", "ghost")
        self.assertEqual(self._last(h)[0], 404)

    def test_worktree_restore_no_snapshot_409(self):
        self._project()
        self._task()
        h = self._h()
        h._task_worktree_restore("u1", "t1")
        code, payload = self._last(h)
        self.assertEqual(code, 409)
        self.assertIn("no snapshot", payload["error"])

    def test_worktree_status_ghost(self):
        h = self._h()
        h._task_worktree_status("u1", "ghost")
        self.assertEqual(self._last(h)[0], 404)

    def test_pr_checks_ghost(self):
        h = self._h()
        h._task_pr_checks("u1", "ghost")
        self.assertEqual(self._last(h)[0], 404)

    def test_pr_checks_git_head_fallback(self):
        self._project(repo_url="https://github.com/o/r")
        self._task(pr_number=7, pr_url="https://github.com/o/r/pull/7")

        def _runner(cmd, cwd=None):
            if cmd and cmd[0] == "git":
                return SimpleNamespace(returncode=0,
                                       stdout="deadbeef\n", stderr="")
            return SimpleNamespace(returncode=0, stdout="", stderr="")
        h = self._h()
        h.server.pr_gh_runner = _runner
        diag = {"checks": [], "all_pass": True, "failed": [],
                "diagnostics": [], "check_runs": {"available": False}}
        with mock.patch("cloud.server._wait_build_diag",
                        return_value=diag):
            h._task_pr_checks("u1", "t1")
        code, payload = self._last(h)
        self.assertEqual(code, 200)
        self.assertEqual(payload["head_sha"], "deadbeef")

    def test_pr_checks_diag_runtime_error_502(self):
        self._project(repo_url="https://github.com/o/r")
        self._task(pr_number=7)
        h = self._h()
        h.server.pr_gh_runner = lambda cmd, cwd=None: SimpleNamespace(
            returncode=0, stdout="", stderr="")
        with mock.patch("cloud.server._wait_build_diag",
                        side_effect=RuntimeError("diag boom")):
            h._task_pr_checks("u1", "t1")
        code, payload = self._last(h)
        self.assertEqual(code, 502)
        self.assertIn("diag boom", payload["error"])


class SleepWaitForkTestCase(_Base):
    def test_sleep_read_error(self):
        h = self._h(err="bad json")
        h._task_sleep("u1", "t1")
        self.assertEqual(self._last(h)[0], 400)

    def test_sleep_ghost(self):
        h = self._h(body={})
        h._task_sleep("u1", "ghost")
        self.assertEqual(self._last(h)[0], 404)

    def test_sleep_seconds_not_number(self):
        self._project()
        self._task()
        h = self._h(body={"seconds": "soon"})
        h._task_sleep("u1", "t1")
        code, payload = self._last(h)
        self.assertEqual(code, 400)
        self.assertIn("seconds must be a number", payload["error"])

    def test_wait_read_error(self):
        h = self._h(err="bad json")
        h._task_wait("u1", "t1")
        self.assertEqual(self._last(h)[0], 400)

    def test_wait_ghost(self):
        h = self._h(body={})
        h._task_wait("u1", "ghost")
        self.assertEqual(self._last(h)[0], 404)

    def test_wait_child_vanishes_not_found(self):
        self._project()
        self._task(tid="anchor")
        self._task(tid="c1", status="completed", parent_task_id="anchor")
        # _db calls: anchor check, child list, child check are real;
        # every later _db (the poll fetches) sees no child row.
        calls = {"n": 0}
        real_db = _conn

        def _db():
            calls["n"] += 1
            if calls["n"] <= 3:
                return real_db(self.db_path)
            return _ProxyConn(real_db(self.db_path),
                              none_select_substr="SELECT * FROM tasks")
        h = self._h(body={"task_ids": ["c1"], "timeout_seconds": 0})
        h._db = _db
        h._task_wait("u1", "anchor")
        code, payload = self._last(h)
        self.assertEqual(code, 200)
        self.assertEqual(payload["progress"][0]["status"], "not_found")

    def test_fork_read_error(self):
        h = self._h(err="bad json")
        h._fork_task("u1", "t1")
        self.assertEqual(self._last(h)[0], 400)

    def test_fork_forbidden(self):
        self._project()
        self._task()
        h = self._h(body={"message": "go"})
        h._fork_task("u9", "t1")
        self.assertEqual(self._last(h)[0], 403)

    def test_fork_message_too_long(self):
        self._project()
        self._task()
        h = self._h(body={"message": "x" * 20001})
        h._fork_task("u1", "t1")
        code, payload = self._last(h)
        self.assertEqual(code, 400)
        self.assertIn("message too long", payload["error"])

    def test_loop_root_broken_chain(self):
        h = self._h()
        root = h._loop_root_task_id({"id": "kid",
                                     "forked_from": "ghost-parent"})
        self.assertEqual(root, "kid")

    def test_loop_root_cycle_20_steps(self):
        self._project()
        self._task(tid="a", forked_from="b")
        self._task(tid="b", forked_from="a")
        h = self._h()
        root = h._loop_root_task_id({"id": "a", "forked_from": "b"})
        self.assertIn(root, ("a", "b"))

    def test_loop_create_subtask_targeted(self):
        self._project()
        h = self._h()
        sub = h._loop_create_subtask(
            {"kind": "fix", "module_id": "m1", "contract_version": "v2",
             "parent_task_id": "p0", "scope": "s",
             "targeted_items": ["item-a"],
             "pushback_package": {"failures": ["f1"],
                                  "reaudit_criteria": ["c1"]}},
            {"id": "p0", "project_id": "p1", "created_by": "u1"}, "u1")
        self.assertIn("Targeted items only", sub["prompt"])
        self.assertEqual(sub["status"], "queued")

    def test_audit_loop_verdict_read_error(self):
        h = self._h(err="bad json")
        h._audit_loop_verdict("u1", "t1")
        self.assertEqual(self._last(h)[0], 400)

    def test_audit_loop_fix_read_error(self):
        h = self._h(err="bad json")
        h._audit_loop_fix("u1", "t1")
        self.assertEqual(self._last(h)[0], 400)

    def test_audit_loop_correction_read_error(self):
        h = self._h(err="bad json")
        h._audit_loop_correction("u1", "t1")
        self.assertEqual(self._last(h)[0], 400)


class ApprovalsResumeRetryTestCase(_Base):
    def test_create_approval_ghost_404(self):
        h = self._h(body={})
        h._create_approval("u1", "ghost")
        self.assertEqual(self._last(h)[0], 404)

    def test_create_approval_read_error(self):
        self._project()
        self._task()
        h = self._h(err="bad json")
        h._create_approval("u1", "t1")
        self.assertEqual(self._last(h)[0], 400)

    def test_create_approval_detail_not_object(self):
        self._project()
        self._task()
        h = self._h(body={"summary": "s", "detail": 123})
        h._create_approval("u1", "t1")
        code, payload = self._last(h)
        self.assertEqual(code, 400)
        self.assertIn("detail must be an object", payload["error"])

    def test_decide_approval_bad_prefix_404(self):
        h = self._h()
        h._decide_approval("u1", "nonsense", "approve")
        self.assertEqual(self._last(h)[0], 404)

    def test_decide_approval_ghost_task_404(self):
        h = self._h()
        h._decide_approval("u1", "ghost/approvals/a1", "approve")
        self.assertEqual(self._last(h)[0], 404)

    def test_goal_resume_ghost_404(self):
        h = self._h()
        h._goal_resume_task("u1", "ghost")
        self.assertEqual(self._last(h)[0], 404)

    def test_goal_resume_quota_bare_return(self):
        self._project()
        self._task(status="interrupted", goal_mode=1)
        h = self._h()
        h._reject_if_over_quota = lambda *a, **k: True
        h._goal_resume_task("u1", "t1")
        self.assertEqual(h.sent, [])

    def test_goal_resume_cas_race_409(self):
        self._project()
        self._task(status="interrupted", goal_mode=1)
        h = self._h()
        h._db = lambda: _ProxyConn(
            _conn(self.db_path),
            rowcount_zero_prefix="UPDATE tasks SET status = 'queued',"
            " goal_resume_pending")
        h._goal_resume_task("u1", "t1")
        code, payload = self._last(h)
        self.assertEqual(code, 409)
        self.assertIn("status changed", payload["error"])

    def test_retry_cas_race_409(self):
        self._project()
        self._task(status="failed")
        h = self._h()
        h._db = lambda: _ProxyConn(
            _conn(self.db_path),
            rowcount_zero_prefix="UPDATE tasks SET status = 'queued',"
            " result_text")
        h._retry_task("u1", "t1")
        code, payload = self._last(h)
        self.assertEqual(code, 409)
        self.assertIn("status changed", payload["error"])


class QuotaAdminTestCase(_Base):
    def test_quota_snapshot_close_swallowed(self):
        from cloud.cost import CostStore as _RealCostStore
        _RealCostStore(self.db_path)
        h = self._h()

        class _Store:
            def __init__(self, db_path):
                self._conn = mock.Mock()
                self._conn.close.side_effect = RuntimeError("down")
        with mock.patch("cloud.server.CostStore", _Store):
            quota, used, unpriced, month = h._quota_snapshot("u1")
        self.assertIsNotNone(month)

    def test_coordinator_reserve_quota_none(self):
        h = self._h()
        h._quota_status = lambda uid: (None, 0, "2026-11")
        self.assertTrue(h._check_coordinator_reserve("u1", 3))

    def test_coordinator_reserve_sufficient(self):
        h = self._h()
        h._quota_status = lambda uid: (10_000_000, 0, "2026-11")
        self.assertTrue(h._check_coordinator_reserve("u1", 2))

    def test_audit_admin_failure_swallowed(self):
        h = self._h()
        with mock.patch("cloud.server._auth_store",
                        side_effect=RuntimeError("store down")):
            self.assertIsNone(h._audit_admin("u1", "x", "y"))

    def test_is_global_maintainer_true(self):
        h = self._h()
        with mock.patch.dict(os.environ, {"CODEX_GLOBAL_ADMINS": "u1"}):
            self.assertTrue(h._is_global_maintainer("u1"))

    def test_cost_by_creator_close_swallowed(self):
        from cloud.cost import CostStore as _RealCostStore
        _RealCostStore(self.db_path)
        h = self._h()

        class _Store:
            def __init__(self, db_path):
                self._conn = mock.Mock()
                self._conn.close.side_effect = RuntimeError("down")

            def dual_column_report_by_creator(self, target, month=None):
                return {}
        with mock.patch("cloud.server.CostStore", _Store):
            payload = h._cost_by_creator_payload("u1", None)
        self.assertEqual(payload["user_id"], "u1")

    def test_admin_quota_get_user_not_found(self):
        self._project()
        h = self._h()
        with mock.patch.dict(os.environ, {"CODEX_GLOBAL_ADMINS": "u1"}):
            h._admin_quota_get("u1", {"user_id": ["ghost"]})
        self.assertEqual(self._last(h)[0], 404)

    def test_admin_ledger_export_non_admin_403(self):
        h = self._h()
        h._admin_ledger_export("u9", {})
        self.assertEqual(self._last(h)[0], 403)

    def test_admin_quota_set_read_error(self):
        h = self._h(err="bad json")
        with mock.patch.dict(os.environ, {"CODEX_GLOBAL_ADMINS": "u1"}):
            h._admin_quota_set("u1")
        self.assertEqual(self._last(h)[0], 400)

    def test_admin_quota_set_bad_quota(self):
        self._project()
        self._user_row("u2")
        h = self._h(body={"user_id": "u2", "quota_usd_micros": "abc"})
        with mock.patch.dict(os.environ, {"CODEX_GLOBAL_ADMINS": "u1"}):
            h._admin_quota_set("u1")
        code, payload = self._last(h)
        self.assertEqual(code, 400)
        self.assertIn("must be an integer or null", payload["error"])

    def test_admin_quota_set_insert_branch(self):
        self._project()
        self._user_row("u2")
        h = self._h(body={"user_id": "u2", "quota_usd_micros": 5000})
        with mock.patch.dict(os.environ, {"CODEX_GLOBAL_ADMINS": "u1"}):
            h._admin_quota_set("u1")
        self.assertEqual(self._last(h)[0], 200)
        c = _conn(self.db_path)
        row = c.execute(
            "SELECT quota_usd_micros FROM user_quotas WHERE user_id = ?",
            ("u2",)).fetchone()
        c.close()
        self.assertEqual(row[0], 5000)

    def test_notify_pr_event_dispatch_raises(self):
        h = self._h()
        with mock.patch("cloud.server._push_dispatch",
                        side_effect=RuntimeError("push down")):
            self.assertIsNone(h._notify_pr_event("p1", "merged", {}))

    def test_notify_pr_event_dispatch_errors(self):
        h = self._h()
        with mock.patch("cloud.server._push_dispatch",
                        return_value={"errors": ["one failed"]}):
            self.assertIsNone(h._notify_pr_event("p1", "merged", {}))

    def test_task_pr_ghost_404(self):
        h = self._h()
        h._task_pr("u1", "ghost")
        self.assertEqual(self._last(h)[0], 404)

    def test_heartbeat_ghost_404(self):
        h = self._h()
        h._heartbeat_task("u1", "ghost")
        self.assertEqual(self._last(h)[0], 404)

    def test_heartbeat_forbidden_403(self):
        self._project()
        self._task()
        h = self._h()
        h._heartbeat_task("u9", "t1")
        self.assertEqual(self._last(h)[0], 403)

    def test_admin_sweep_read_error(self):
        h = self._h(err="bad json")
        with mock.patch.dict(os.environ, {"CODEX_GLOBAL_ADMINS": "u1"}):
            h._admin_sweep("u1")
        self.assertEqual(self._last(h)[0], 400)

    def test_admin_sweep_bad_threshold(self):
        h = self._h(body={"threshold_seconds": "soon"})
        with mock.patch.dict(os.environ, {"CODEX_GLOBAL_ADMINS": "u1"}):
            h._admin_sweep("u1")
        code, payload = self._last(h)
        self.assertEqual(code, 400)
        self.assertIn("must be an integer", payload["error"])


class MetricsOauthAppTestCase(_Base):
    def _cost_rows(self):
        from cloud.cost import CostStore
        CostStore(self.db_path)
        c = _conn(self.db_path)
        c.execute(
            "INSERT INTO cost_records(task_id, project_id, model,"
            " total_tokens, usd_micros, source, fallback_count)"
            " VALUES(?, ?, ?, ?, ?, ?, ?)",
            ("t1", "p1", "m1", 100, 5000, "actual", 0))
        c.execute(
            "INSERT INTO cost_records(task_id, project_id, model,"
            " total_tokens, usd_micros, source, fallback_count)"
            " VALUES(?, ?, ?, ?, ?, ?, ?)",
            ("t9", "p2", "m1", 50, 100, "actual", 0))
        c.execute(
            "INSERT INTO cost_records(task_id, project_id, model,"
            " total_tokens, usd_micros, source, fallback_count)"
            " VALUES(?, ?, ?, ?, ?, ?, ?)",
            ("t1", "p1", "m1", 70, None, "actual", 0))
        c.commit()
        c.close()

    def test_metrics_non_member_skip_and_priced(self):
        self._project()
        self._project(pid="p2", owner="u2")
        self._task()
        self._cost_rows()
        h = self._h()
        h._metrics("u1")
        code, payload = self._last(h)
        self.assertEqual(code, 200)
        self.assertEqual(payload["cost"]["actual_usd_micros"], 5000)

    def test_metrics_approval_timeout_failure_none(self):
        self._project()
        h = self._h()
        with mock.patch("cloud.server._appr_timeout_stats",
                        side_effect=RuntimeError("stats down")):
            h._metrics("u1")
        code, payload = self._last(h)
        self.assertEqual(code, 200)
        self.assertIsNone(payload["approval_timeout"])

    def test_metrics_workers_failure_empty(self):
        self._project()
        h = self._h()
        with mock.patch("cloud.server._registry_list",
                        side_effect=RuntimeError("registry down")):
            h._metrics("u1")
        code, payload = self._last(h)
        self.assertEqual(code, 200)
        self.assertEqual(payload["workers"]["total"], 0)

    def test_metrics_report_unpriced_and_skip(self):
        self._project()
        self._project(pid="p2", owner="u2")
        self._task()
        self._cost_rows()
        h = self._h()
        h._metrics_report("u1")
        self.assertEqual(self._last(h)[0], 200)

    def test_oauth_callback_provider_none(self):
        h = self._h()
        h._oauth_callback("/oauth/callback")
        self.assertEqual(self._last(h)[0], 404)

    def test_oauth_callback_read_error(self):
        h = self._h(err="bad json")
        h._oauth_callback("/oauth/github/callback")
        self.assertEqual(self._last(h)[0], 400)

    def test_oauth_callback_get_provider_none(self):
        h = self._h()
        h.path = "/oauth/callback"
        h._oauth_callback_get("/oauth/callback")
        self.assertEqual(h.html[0][0], 404)

    def test_oauth_callback_get_missing_params(self):
        h = self._h()
        h.path = "/oauth/github/callback?state=x"
        h._oauth_callback_get("/oauth/github/callback")
        self.assertEqual(h.html[0][0], 400)

    def test_oauth_callback_get_bind_error(self):
        h = self._h()
        h.path = "/oauth/github/callback?code=c&state=s"
        h._oauth_finish = mock.Mock(
            side_effect=BindError("taken", reason="identity_bound"))
        h._oauth_callback_get("/oauth/github/callback")
        self.assertEqual(h.html[0][0], 409)

    def test_oauth_bindings_get_unauthorized(self):
        h = self._h()
        h._authenticate = lambda: None
        h._oauth_bindings_get("github")
        self.assertEqual(self._last(h)[0], 401)

    def test_oauth_bindings_delete_unauthorized(self):
        h = self._h()
        h._authenticate = lambda: None
        h._oauth_bindings_delete("github")
        self.assertEqual(self._last(h)[0], 401)

    def test_github_app_setup_state_body_default(self):
        h = self._h(err="bad json")
        h._authenticate = lambda: "u1"
        with mock.patch.dict(os.environ, {
                "CODEX_GLOBAL_ADMINS": "u1",
                "GITHUB_APP_PUBLIC_BASE": "https://api.example.test"}):
            h._github_app_setup_state()
        code, payload = self._last(h)
        self.assertEqual(code, 200)
        self.assertIn("setup_url", payload)

    def test_github_app_setup_state_close_swallowed(self):
        h = self._h(body={})
        h._authenticate = lambda: "u1"

        class _Store:
            def __init__(self, db_path):
                self.conn = mock.Mock()
                self.conn.close.side_effect = RuntimeError("down")

            def create_state(self, issued_by=None):
                return "st1"
        with mock.patch.dict(os.environ, {
                "CODEX_GLOBAL_ADMINS": "u1",
                "GITHUB_APP_PUBLIC_BASE": "https://api.example.test"}), \
                mock.patch("cloud.handler_oauth.SetupStateStore", _Store):
            h._github_app_setup_state()
        self.assertEqual(self._last(h)[0], 200)

    def test_github_app_setup_state_org_url(self):
        h = self._h(body={"org": "acme"})
        h._authenticate = lambda: "u1"
        with mock.patch.dict(os.environ, {
                "CODEX_GLOBAL_ADMINS": "u1",
                "GITHUB_APP_PUBLIC_BASE": "https://api.example.test"}):
            h._github_app_setup_state()
        code, payload = self._last(h)
        self.assertEqual(code, 200)
        self.assertIn("org=acme", payload["setup_url"])

    def test_github_app_setup_page_not_configured(self):
        h = self._h()
        h.path = "/github-app/setup?state=st1"
        with mock.patch.dict(os.environ, {}, clear=False):
            os.environ.pop("GITHUB_APP_PUBLIC_BASE", None)
            h._github_app_setup_page()
        self.assertEqual(h.html[0][0], 400)

    def test_github_app_callback_missing_params(self):
        h = self._h()
        h.path = "/github-app/callback"
        h._github_app_callback()
        self.assertEqual(h.html[0][0], 400)

    def test_github_app_callback_exchange_error(self):
        h = self._h()
        h.path = "/github-app/callback?code=c&state=st1"

        class _Store:
            def __init__(self, db_path):
                self.conn = mock.Mock()
                self.conn.close.side_effect = RuntimeError("down")

            def state_issuer(self, state):
                return "u1"

            def consume_state(self, state, allowed_issuers=None):
                return True
        from cloud.github_app import AppError
        with mock.patch("cloud.handler_oauth.SetupStateStore", _Store), \
                mock.patch("cloud.handler_oauth.exchange_manifest_code",
                           side_effect=AppError("bad code",
                                                reason="exchange_failed")):
            h._github_app_callback()
        self.assertEqual(h.html[0][0], 400)

    def test_github_app_status_unauthorized(self):
        h = self._h()
        h._authenticate = lambda: None
        h._github_app_status()
        self.assertEqual(self._last(h)[0], 401)


class FullAutoKeysPushTestCase(_Base):
    def test_full_auto_config_plan_error(self):
        h = self._h()
        with mock.patch("cloud.handler_fullauto.auto_config_plan",
                        side_effect=FullAutoError("boom", reason="x")):
            h._full_auto_config_plan("u1", {"repo": ["o/r"]})
        code, payload = self._last(h)
        self.assertEqual(code, 400)
        self.assertEqual(payload["reason"], "x")

    def test_full_auto_validate_read_error(self):
        h = self._h(err="bad json")
        h._full_auto_validate_action("u1")
        self.assertEqual(self._last(h)[0], 400)

    def test_full_auto_validate_action_required(self):
        h = self._h(body={})
        h._full_auto_validate_action("u1")
        code, payload = self._last(h)
        self.assertEqual(code, 400)
        self.assertIn("action required", payload["error"])

    def test_full_auto_merge_gate_read_error(self):
        h = self._h(err="bad json")
        h._full_auto_merge_gate("u1")
        self.assertEqual(self._last(h)[0], 400)

    def test_full_auto_ci_fix_read_error(self):
        h = self._h(err="bad json")
        h._full_auto_ci_fix("u1")
        self.assertEqual(self._last(h)[0], 400)

    def test_full_auto_ci_fix_bad_attempt(self):
        h = self._h(body={"attempt": "x"})
        h._full_auto_ci_fix("u1")
        code, payload = self._last(h)
        self.assertEqual(code, 400)
        self.assertIn("attempt must be an integer", payload["error"])

    def test_full_auto_release_read_error(self):
        h = self._h(err="bad json")
        h._full_auto_release_plan("u1")
        self.assertEqual(self._last(h)[0], 400)

    def test_full_auto_release_tag_required(self):
        h = self._h(body={})
        h._full_auto_release_plan("u1")
        code, payload = self._last(h)
        self.assertEqual(code, 400)
        self.assertIn("tag required", payload["error"])

    def test_full_auto_rollback_read_error(self):
        h = self._h(err="bad json")
        h._full_auto_rollback_plan("u1")
        self.assertEqual(self._last(h)[0], 400)

    def test_full_auto_audit_limit_fallback(self):
        h = self._h()
        with mock.patch.dict(os.environ, {"CODEX_GLOBAL_ADMINS": "u1"}):
            h._full_auto_audit_list("u1", {"limit": ["abc"]})
        self.assertEqual(self._last(h)[0], 200)

    def test_full_auto_audit_query_failure_empty(self):
        h = self._h()
        h._db = lambda: _ProxyConn(
            _conn(self.db_path),
            fail_select_substr="FROM audit_log",
            fail_exc=RuntimeError("audit down"))
        with mock.patch.dict(os.environ, {"CODEX_GLOBAL_ADMINS": "u1"}):
            h._full_auto_audit_list("u1", {})
        code, payload = self._last(h)
        self.assertEqual(code, 200)
        self.assertEqual(payload["audit"], [])

    def test_user_keys_create_body_default(self):
        self._user_row("u1")
        h = self._h(err="bad json")
        h._authenticate = lambda: "u1"
        h._user_keys_create()
        code, payload = self._last(h)
        self.assertEqual(code, 201)
        self.assertEqual(payload["label"], "api-key")

    def test_user_keys_revoke_unauthorized(self):
        h = self._h()
        h._authenticate = lambda: None
        h._user_keys_revoke("k1")
        self.assertEqual(self._last(h)[0], 401)

    def test_user_keys_revoke_others_unauthorized(self):
        h = self._h()
        h._authenticate = lambda: None
        h._user_keys_revoke_others()
        self.assertEqual(self._last(h)[0], 401)

    def test_push_create_read_error(self):
        h = self._h(err="bad json")
        h._push_create_endpoint("u1", "p1")
        self.assertEqual(self._last(h)[0], 400)

    def test_push_create_bad_channel(self):
        h = self._h(body={"channel": "bogus", "target": "t"})
        h._push_create_endpoint("u1", "p1")
        self.assertEqual(self._last(h)[0], 400)

    def test_push_dispatch_read_error(self):
        h = self._h(err="bad json")
        h._push_dispatch_endpoint("u1")
        self.assertEqual(self._last(h)[0], 400)

    def test_push_dispatch_missing_fields(self):
        h = self._h(body={})
        h._push_dispatch_endpoint("u1")
        code, payload = self._last(h)
        self.assertEqual(code, 400)
        self.assertIn("event_type and project_id required",
                      payload["error"])


class AutomationTodosTestCase(_Base):
    def _fail_db(self, h):
        h._db = lambda: _FailAllConn(_conn(self.db_path))

    def test_automation_list_store_failure_502(self):
        h = self._h()
        self._fail_db(h)
        h._automation_list("p1")
        self.assertEqual(self._last(h)[0], 502)

    def test_todos_list_store_failure_502(self):
        h = self._h()
        self._fail_db(h)
        h._todos_list("p1", {})
        self.assertEqual(self._last(h)[0], 502)

    def test_todos_create_store_failure_502(self):
        h = self._h(body={"title": "t"})
        self._fail_db(h)
        h._todos_create("u1", "p1")
        self.assertEqual(self._last(h)[0], 502)

    def test_todos_get_store_failure_502(self):
        h = self._h()
        self._fail_db(h)
        h._todos_get("p1", "td1")
        self.assertEqual(self._last(h)[0], 502)

    def test_todos_delete_store_failure_502(self):
        h = self._h()
        self._fail_db(h)
        h._todos_delete("u1", "p1", "td1")
        self.assertEqual(self._last(h)[0], 502)

    def test_automation_cancel_store_failure_502(self):
        h = self._h()
        self._fail_db(h)
        h._automation_cancel("u1", "p1", "a1")
        self.assertEqual(self._last(h)[0], 502)

    def test_automation_get_ghost_404(self):
        h = self._h()
        h._automation_get("u1", "p1", "ghost")
        self.assertEqual(self._last(h)[0], 404)

    def test_automation_get_runs_failure_note(self):
        from cloud.automations import create_automation
        c = _conn(self.db_path)
        auto = create_automation(c, "p1", "nightly", "do it", "u1")
        c.close()
        h = self._h()
        h._db = lambda: _ProxyConn(
            _conn(self.db_path),
            fail_select_substr="FROM automation_runs",
            fail_exc=RuntimeError("runs down"))
        h._automation_get("u1", "p1", auto["id"])
        code, payload = self._last(h)
        self.assertEqual(code, 200)
        self.assertEqual(payload["runs_note"], "runs unavailable")

    def test_automation_bind_get_ghost_404(self):
        h = self._h()
        h._automation_bind_get("u1", "p1", "ghost")
        self.assertEqual(self._last(h)[0], 404)

    def test_automation_unbind_ghost_404(self):
        h = self._h()
        h._automation_unbind("u1", "p1", "ghost")
        self.assertEqual(self._last(h)[0], 404)

    def test_automation_unbind_no_binding_404(self):
        from cloud.automations import create_automation
        c = _conn(self.db_path)
        auto = create_automation(c, "p1", "nightly", "do it", "u1")
        c.close()
        h = self._h()
        h._automation_unbind("u1", "p1", auto["id"])
        code, payload = self._last(h)
        self.assertEqual(code, 404)
        self.assertIn("no binding", payload["error"])

    def test_triage_store_failure_502(self):
        h = self._h()
        self._fail_db(h)
        h._triage_inbox("p1")
        self.assertEqual(self._last(h)[0], 502)