import { useCallback, useEffect, useRef, useState } from "react"; import { useConsole } from "../../context"; import { api, getKey } from "../../api/client"; import { paths } from "../../api/paths"; // Browser remote view: watches the agent's Chromium live and, after // takeover, forwards the user's clicks/keys to it. // // Transport note: EventSource cannot send the X-API-Key header, so // the SSE stream is read with fetch + a ReadableStream reader. // The stream carries base64 jpeg frames; the image element shows // the latest frame. Click coordinates are scaled from the displayed // size back to the 1280x800 viewport the tools use. type BrowserState = { url?: string; title?: string; holder?: string; started?: boolean; streaming?: boolean; }; export default function BrowserPanel() { const ctx = useConsole(); const t = ctx.currentTask; const [img, setImg] = useState(""); const [state, setState] = useState(null); const [status, setStatus] = useState("未开始"); const [err, setErr] = useState(""); const [watching, setWatching] = useState(false); const abortRef = useRef(null); const imgRef = useRef(null); const refreshState = useCallback(async () => { if (!t) return; try { const s = await api(paths.taskBrowserState(t.id)); setState(s); setErr(""); } catch (e) { const msg = e instanceof Error ? e.message : String(e); if (msg.includes("503")) setState(null); else setErr(msg); } }, [t]); useEffect(() => { setImg(""); setState(null); setStatus("未开始"); setErr(""); setWatching(false); abortRef.current?.abort(); if (!t) return; void refreshState(); return () => { abortRef.current?.abort(); }; }, [t, refreshState]); // While watching, the stream payload already carries holder/url/ // title at frame rate (plus a 5s heartbeat), so the 5s state // poll would be pure redundancy -- run it only while not // watching, so the header still goes fresh when a session // starts before the user opens the stream. useEffect(() => { if (!t || watching) return; const timer = setInterval(() => void refreshState(), 5000); return () => clearInterval(timer); }, [t, watching, refreshState]); const startWatch = useCallback(async () => { if (!t) return; abortRef.current?.abort(); const ctrl = new AbortController(); abortRef.current = ctrl; setWatching(true); setStatus("连接中…"); setErr(""); try { const r = await fetch(paths.taskBrowserStream(t.id), { headers: { "X-API-Key": getKey() }, signal: ctrl.signal, }); if (!r.ok || !r.body) { const d = await r.json().catch(() => null) as { error?: string } | null; throw new Error((d && d.error) || "HTTP " + r.status); } setStatus("直播中"); const reader = r.body.getReader(); const decoder = new TextDecoder(); let buf = ""; for (;;) { const { done, value } = await reader.read(); if (done) break; buf += decoder.decode(value, { stream: true }); let idx = buf.indexOf("\n\n"); while (idx >= 0) { const block = buf.slice(0, idx); buf = buf.slice(idx + 2); for (const line of block.split("\n")) { if (!line.startsWith("data: ")) continue; try { const payload = JSON.parse(line.slice(6)) as { image?: string; holder?: string; url?: string; title?: string; }; if (payload.image) { setImg("data:image/jpeg;base64," + payload.image); } const patch: BrowserState = {}; if (payload.holder) patch.holder = payload.holder; if (payload.url) patch.url = payload.url; if (payload.title !== undefined) patch.title = payload.title; if (Object.keys(patch).length > 0) { setState((s) => ({ ...(s || {}), ...patch })); } } catch { // ignore a torn frame line } } idx = buf.indexOf("\n\n"); } } setStatus("流已断开"); } catch (e) { if ((e as Error).name === "AbortError") return; setErr(e instanceof Error ? e.message : String(e)); setStatus("未开始"); } finally { setWatching(false); } }, [t]); const stopWatch = useCallback(() => { abortRef.current?.abort(); setWatching(false); setStatus("已停止"); }, []); const takeover = useCallback(async (holder: "user" | "agent") => { if (!t) return; try { const s = await api( paths.taskBrowserControl(t.id), "POST", { holder }); setState(s); setErr(""); } catch (e) { setErr(e instanceof Error ? e.message : String(e)); } }, [t]); const clickImage = useCallback(async (e: React.MouseEvent) => { if (!t || state?.holder !== "user" || !imgRef.current) return; const rect = imgRef.current.getBoundingClientRect(); const x = Math.round(((e.clientX - rect.left) / rect.width) * 1280); const y = Math.round(((e.clientY - rect.top) / rect.height) * 800); try { await api(paths.taskBrowserInput(t.id), "POST", { type: "mouse_click", x, y, }); } catch (err) { setErr(err instanceof Error ? err.message : String(err)); } }, [t, state]); const keyDown = useCallback(async (e: React.KeyboardEvent) => { if (!t || state?.holder !== "user") return; e.preventDefault(); try { if (e.key.length === 1) { await api(paths.taskBrowserInput(t.id), "POST", { type: "type", text: e.key, }); } else { await api(paths.taskBrowserInput(t.id), "POST", { type: "key", key: e.key, }); } } catch (err) { setErr(err instanceof Error ? err.message : String(err)); } }, [t, state]); if (!t) return
点任务行查看详情
; const holder = state?.holder || "—"; const active = state !== null; return (
状态 {status} 会话 {active ? "活跃" : "未激活(轮次结束即消失)"} 控制权 {holder} {state?.url ? {state.url.slice(0, 80)} : null}
{!watching ? : } {holder !== "user" ? : }
{err ?
{err}
: null} {holder === "user" ? (
你已接管:点画面即点击,键盘输入直达页面。交还后 Agent 续跑前会先快照确认你的改动。
) : null}
{img ? ( 浏览器实时画面 void clickImage(e)} onKeyDown={(e) => void keyDown(e)} style={{ width: "100%", maxWidth: 1280, cursor: holder === "user" ? "crosshair" : "default", border: "1px solid var(--border, #ccc)", }} /> ) : (
{active ? "点“开始观看”接入画面流(接管后点画面即点击)" : "浏览器会话未激活:任务本轮未开浏览器,或轮次已结束。"}
)}
); }