codex-app-server/cloud/console-react/src/sections/detail/BrowserPanel.tsx

import { useCallback, useEffect, useRef, useState } from "react";
import { useConsole } from "../../context";
import { api, getKey } from "../../api/client";
import { paths } from "../../api/paths";

// Browser remote view: watches the agent's Chromium live and, after
// takeover, forwards the user's clicks/keys to it.
//
// Transport note: EventSource cannot send the X-API-Key header, so
// the SSE stream is read with fetch + a ReadableStream reader.
// The stream carries base64 jpeg frames; the image element shows
// the latest frame. Click coordinates are scaled from the displayed
// size back to the 1280x800 viewport the tools use.
type BrowserState = {
  url?: string;
  title?: string;
  holder?: string;
  started?: boolean;
  streaming?: boolean;
};

export default function BrowserPanel() {
  const ctx = useConsole();
  const t = ctx.currentTask;
  const [img, setImg] = useState("");
  const [state, setState] = useState<BrowserState | null>(null);
  const [status, setStatus] = useState("未开始");
  const [err, setErr] = useState("");
  const [watching, setWatching] = useState(false);
  const abortRef = useRef<AbortController | null>(null);
  const imgRef = useRef<HTMLImageElement | null>(null);

  const refreshState = useCallback(async () => {
    if (!t) return;
    try {
      const s = await api<BrowserState>(paths.taskBrowserState(t.id));
      setState(s);
      setErr("");
    } catch (e) {
      const msg = e instanceof Error ? e.message : String(e);
      if (msg.includes("503")) setState(null);
      else setErr(msg);
    }
  }, [t]);

  useEffect(() => {
    setImg("");
    setState(null);
    setStatus("未开始");
    setErr("");
    setWatching(false);
    abortRef.current?.abort();
    if (!t) return;
    void refreshState();
    return () => {
      abortRef.current?.abort();
    };
  }, [t, refreshState]);

  // While watching, the stream payload already carries holder/url/
  // title at frame rate (plus a 5s heartbeat), so the 5s state
  // poll would be pure redundancy -- run it only while not
  // watching, so the header still goes fresh when a session
  // starts before the user opens the stream.
  useEffect(() => {
    if (!t || watching) return;
    const timer = setInterval(() => void refreshState(), 5000);
    return () => clearInterval(timer);
  }, [t, watching, refreshState]);

  const startWatch = useCallback(async () => {
    if (!t) return;
    abortRef.current?.abort();
    const ctrl = new AbortController();
    abortRef.current = ctrl;
    setWatching(true);
    setStatus("连接中…");
    setErr("");
    try {
      const r = await fetch(paths.taskBrowserStream(t.id), {
        headers: { "X-API-Key": getKey() },
        signal: ctrl.signal,
      });
      if (!r.ok || !r.body) {
        const d = await r.json().catch(() => null) as { error?: string } | null;
        throw new Error((d && d.error) || "HTTP " + r.status);
      }
      setStatus("直播中");
      const reader = r.body.getReader();
      const decoder = new TextDecoder();
      let buf = "";
      for (;;) {
        const { done, value } = await reader.read();
        if (done) break;
        buf += decoder.decode(value, { stream: true });
        let idx = buf.indexOf("\n\n");
        while (idx >= 0) {
          const block = buf.slice(0, idx);
          buf = buf.slice(idx + 2);
          for (const line of block.split("\n")) {
            if (!line.startsWith("data: ")) continue;
            try {
              const payload = JSON.parse(line.slice(6)) as {
                image?: string;
                holder?: string;
                url?: string;
                title?: string;
              };
              if (payload.image) {
                setImg("data:image/jpeg;base64," + payload.image);
              }
              const patch: BrowserState = {};
              if (payload.holder) patch.holder = payload.holder;
              if (payload.url) patch.url = payload.url;
              if (payload.title !== undefined) patch.title = payload.title;
              if (Object.keys(patch).length > 0) {
                setState((s) => ({ ...(s || {}), ...patch }));
              }
            } catch {
              // ignore a torn frame line
            }
          }
          idx = buf.indexOf("\n\n");
        }
      }
      setStatus("流已断开");
    } catch (e) {
      if ((e as Error).name === "AbortError") return;
      setErr(e instanceof Error ? e.message : String(e));
      setStatus("未开始");
    } finally {
      setWatching(false);
    }
  }, [t]);

  const stopWatch = useCallback(() => {
    abortRef.current?.abort();
    setWatching(false);
    setStatus("已停止");
  }, []);

  const takeover = useCallback(async (holder: "user" | "agent") => {
    if (!t) return;
    try {
      const s = await api<BrowserState>(
        paths.taskBrowserControl(t.id), "POST", { holder });
      setState(s);
      setErr("");
    } catch (e) {
      setErr(e instanceof Error ? e.message : String(e));
    }
  }, [t]);

  const clickImage = useCallback(async (e: React.MouseEvent) => {
    if (!t || state?.holder !== "user" || !imgRef.current) return;
    const rect = imgRef.current.getBoundingClientRect();
    const x = Math.round(((e.clientX - rect.left) / rect.width) * 1280);
    const y = Math.round(((e.clientY - rect.top) / rect.height) * 800);
    try {
      await api(paths.taskBrowserInput(t.id), "POST", {
        type: "mouse_click", x, y,
      });
    } catch (err) {
      setErr(err instanceof Error ? err.message : String(err));
    }
  }, [t, state]);

  const keyDown = useCallback(async (e: React.KeyboardEvent) => {
    if (!t || state?.holder !== "user") return;
    e.preventDefault();
    try {
      if (e.key.length === 1) {
        await api(paths.taskBrowserInput(t.id), "POST", {
          type: "type", text: e.key,
        });
      } else {
        await api(paths.taskBrowserInput(t.id), "POST", {
          type: "key", key: e.key,
        });
      }
    } catch (err) {
      setErr(err instanceof Error ? err.message : String(err));
    }
  }, [t, state]);

  if (!t) return <div className="muted">点任务行查看详情</div>;
  const holder = state?.holder || "—";
  const active = state !== null;
  return (
    <div>
      <div style={{ display: "flex", gap: 8, flexWrap: "wrap", alignItems: "center" }}>
        <span>状态 <b>{status}</b></span>
        <span className="muted">会话 {active ? "活跃" : "未激活（轮次结束即消失）"}</span>
        <span className="muted">控制权 {holder}</span>
        {state?.url ? <span className="muted" title={state.url}>{state.url.slice(0, 80)}</span> : null}
      </div>
      <div style={{ display: "flex", gap: 8, flexWrap: "wrap", marginTop: 8 }}>
        {!watching
          ? <button type="button" onClick={() => void startWatch()}>开始观看</button>
          : <button type="button" onClick={stopWatch}>停止观看</button>}
        <button type="button" onClick={() => void refreshState()}>刷新状态</button>
        {holder !== "user"
          ? <button type="button" onClick={() => void takeover("user")} disabled={!active}>接管</button>
          : <button type="button" onClick={() => void takeover("agent")}>交还给 Agent</button>}
      </div>
      {err ? <div className="muted" style={{ marginTop: 8 }}>{err}</div> : null}
      {holder === "user" ? (
        <div className="muted" style={{ marginTop: 8 }}>
          你已接管：点画面即点击，键盘输入直达页面。交还后 Agent 续跑前会先快照确认你的改动。
        </div>
      ) : null}
      <div style={{ marginTop: 8 }}>
        {img ? (
          <img
            ref={imgRef}
            src={img}
            alt="浏览器实时画面"
            tabIndex={0}
            onClick={(e) => void clickImage(e)}
            onKeyDown={(e) => void keyDown(e)}
            style={{
              width: "100%",
              maxWidth: 1280,
              cursor: holder === "user" ? "crosshair" : "default",
              border: "1px solid var(--border, #ccc)",
            }}
          />
        ) : (
          <div className="muted">
            {active
              ? "点“开始观看”接入画面流（接管后点画面即点击）"
              : "浏览器会话未激活：任务本轮未开浏览器，或轮次已结束。"}
          </div>
        )}
      </div>
    </div>
  );
}