feat: 控制台浏览器远控面板
- TaskDetailPanel 新增“浏览器”页签，内嵌 BrowserPanel。 - 开始观看：fetch + ReadableStream 读 SSE（EventSource 发不出 X-API-Key），最新帧以 data URI 显示；状态每 5 秒轮询。 - 接管/交还：POST /browser/control；接管后点画面按 1280x800 换算坐标 POST /browser/input，键盘直达页面。 - 产物：cloud/console.html 由 vite 单文件构建重打（341451 字节， 含 browser/state 与“浏览器远控”标记）。
This commit is contained in:
host committed
1 parent
80b32f25b3
commit
7589876cff
4 files changed
+259
-24
No files matched your search
@@ -56,6 +56,10 @@ export const paths = {
|
||||
taskThreads: (id: string) => "/tasks/" + enc(id) + "/threads",
|
||||
taskThreadAction: (id: string, kind: string) => "/tasks/" + enc(id) + "/thread/" + kind,
|
||||
taskCost: (id: string) => "/tasks/" + enc(id) + "/cost",
|
||||
taskBrowserState: (id: string) => "/tasks/" + enc(id) + "/browser/state",
|
||||
taskBrowserStream: (id: string) => "/tasks/" + enc(id) + "/browser/stream",
|
||||
taskBrowserInput: (id: string) => "/tasks/" + enc(id) + "/browser/input",
|
||||
taskBrowserControl: (id: string) => "/tasks/" + enc(id) + "/browser/control",
|
||||
uploads: "/uploads",
|
||||
adminQuotas: "/admin/quotas",
|
||||
adminQuotaUser: (uid: string) => "/admin/quotas?user_id=" + enc(uid),
|
||||
|
||||
@@ -5,6 +5,7 @@ import { shortId, fmt } from "../api/client";
|
||||
import { STATUS_EXPLAIN } from "../api/types";
|
||||
import { firstChar, durationOf, tokenOf, fmtCostUsd } from "../api/taskMeta";
|
||||
import DetailActions from "./detail/DetailActions";
|
||||
import BrowserPanel from "./detail/BrowserPanel";
|
||||
import ReviewPanel from "./detail/ReviewPanel";
|
||||
import { CoordPanel, AgentsPanel, ApprovalPanel, PeerPanel, ThreadPanel } from "./detail/DetailLists";
|
||||
|
||||
@@ -19,9 +20,9 @@ import { CoordPanel, AgentsPanel, ApprovalPanel, PeerPanel, ThreadPanel } from "
|
||||
// 线程：线程管理
|
||||
// Data, actions and sub-sections are unchanged; only grouping
|
||||
// and visibility changed.
|
||||
type DetailTab = "overview" | "collab" | "review" | "threads";
|
||||
type DetailTab = "overview" | "collab" | "review" | "threads" | "browser";
|
||||
const DETAIL_TABS: Array<[DetailTab, string]> = [
|
||||
["overview", "概览"], ["collab", "协作"], ["review", "评审与审批"], ["threads", "线程"],
|
||||
["overview", "概览"], ["collab", "协作"], ["review", "评审与审批"], ["threads", "线程"], ["browser", "浏览器"],
|
||||
];
|
||||
export default function TaskDetailPanel() {
|
||||
const ctx = useConsole();
|
||||
@@ -104,6 +105,8 @@ export default function TaskDetailPanel() {
|
||||
</>) : tab === "review" ? (<>
|
||||
<Card title="评审队列"><ReviewPanel /></Card>
|
||||
<Card title="运行时审批队列"><ApprovalPanel /></Card>
|
||||
</>) : tab === "browser" ? (<>
|
||||
<Card title="浏览器远控"><BrowserPanel /></Card>
|
||||
</>) : (<>
|
||||
<Card title="线程"><ThreadPanel /></Card>
|
||||
</>)}
|
||||
|
||||
@@ -0,0 +1,223 @@
|
||||
import { useCallback, useEffect, useRef, useState } from "react";
|
||||
import { useConsole } from "../../context";
|
||||
import { api, getKey } from "../../api/client";
|
||||
import { paths } from "../../api/paths";
|
||||
|
||||
// Browser remote view: watches the agent's Chromium live and, after
|
||||
// takeover, forwards the user's clicks/keys to it.
|
||||
//
|
||||
// Transport note: EventSource cannot send the X-API-Key header, so
|
||||
// the SSE stream is read with fetch + a ReadableStream reader.
|
||||
// The stream carries base64 jpeg frames; the image element shows
|
||||
// the latest frame. Click coordinates are scaled from the displayed
|
||||
// size back to the 1280x800 viewport the tools use.
|
||||
type BrowserState = {
|
||||
url?: string;
|
||||
title?: string;
|
||||
holder?: string;
|
||||
started?: boolean;
|
||||
streaming?: boolean;
|
||||
};
|
||||
|
||||
export default function BrowserPanel() {
|
||||
const ctx = useConsole();
|
||||
const t = ctx.currentTask;
|
||||
const [img, setImg] = useState("");
|
||||
const [state, setState] = useState<BrowserState | null>(null);
|
||||
const [status, setStatus] = useState("未开始");
|
||||
const [err, setErr] = useState("");
|
||||
const [watching, setWatching] = useState(false);
|
||||
const abortRef = useRef<AbortController | null>(null);
|
||||
const imgRef = useRef<HTMLImageElement | null>(null);
|
||||
|
||||
const refreshState = useCallback(async () => {
|
||||
if (!t) return;
|
||||
try {
|
||||
const s = await api<BrowserState>(paths.taskBrowserState(t.id));
|
||||
setState(s);
|
||||
setErr("");
|
||||
} catch (e) {
|
||||
const msg = e instanceof Error ? e.message : String(e);
|
||||
if (msg.includes("503")) setState(null);
|
||||
else setErr(msg);
|
||||
}
|
||||
}, [t]);
|
||||
|
||||
useEffect(() => {
|
||||
setImg("");
|
||||
setState(null);
|
||||
setStatus("未开始");
|
||||
setErr("");
|
||||
setWatching(false);
|
||||
abortRef.current?.abort();
|
||||
if (!t) return;
|
||||
void refreshState();
|
||||
const timer = setInterval(() => void refreshState(), 5000);
|
||||
return () => {
|
||||
clearInterval(timer);
|
||||
abortRef.current?.abort();
|
||||
};
|
||||
}, [t, refreshState]);
|
||||
|
||||
const startWatch = useCallback(async () => {
|
||||
if (!t) return;
|
||||
abortRef.current?.abort();
|
||||
const ctrl = new AbortController();
|
||||
abortRef.current = ctrl;
|
||||
setWatching(true);
|
||||
setStatus("连接中…");
|
||||
setErr("");
|
||||
try {
|
||||
const r = await fetch(paths.taskBrowserStream(t.id), {
|
||||
headers: { "X-API-Key": getKey() },
|
||||
signal: ctrl.signal,
|
||||
});
|
||||
if (!r.ok || !r.body) {
|
||||
const d = await r.json().catch(() => null) as { error?: string } | null;
|
||||
throw new Error((d && d.error) || "HTTP " + r.status);
|
||||
}
|
||||
setStatus("直播中");
|
||||
const reader = r.body.getReader();
|
||||
const decoder = new TextDecoder();
|
||||
let buf = "";
|
||||
for (;;) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
buf += decoder.decode(value, { stream: true });
|
||||
let idx = buf.indexOf("\n\n");
|
||||
while (idx >= 0) {
|
||||
const block = buf.slice(0, idx);
|
||||
buf = buf.slice(idx + 2);
|
||||
for (const line of block.split("\n")) {
|
||||
if (!line.startsWith("data: ")) continue;
|
||||
try {
|
||||
const payload = JSON.parse(line.slice(6)) as {
|
||||
image?: string;
|
||||
holder?: string;
|
||||
};
|
||||
if (payload.image) {
|
||||
setImg("data:image/jpeg;base64," + payload.image);
|
||||
}
|
||||
if (payload.holder) {
|
||||
setState((s) => ({ ...(s || {}), holder: payload.holder }));
|
||||
}
|
||||
} catch {
|
||||
// ignore a torn frame line
|
||||
}
|
||||
}
|
||||
idx = buf.indexOf("\n\n");
|
||||
}
|
||||
}
|
||||
setStatus("流已断开");
|
||||
} catch (e) {
|
||||
if ((e as Error).name === "AbortError") return;
|
||||
setErr(e instanceof Error ? e.message : String(e));
|
||||
setStatus("未开始");
|
||||
} finally {
|
||||
setWatching(false);
|
||||
}
|
||||
}, [t]);
|
||||
|
||||
const stopWatch = useCallback(() => {
|
||||
abortRef.current?.abort();
|
||||
setWatching(false);
|
||||
setStatus("已停止");
|
||||
}, []);
|
||||
|
||||
const takeover = useCallback(async (holder: "user" | "agent") => {
|
||||
if (!t) return;
|
||||
try {
|
||||
const s = await api<BrowserState>(
|
||||
paths.taskBrowserControl(t.id), "POST", { holder });
|
||||
setState(s);
|
||||
setErr("");
|
||||
} catch (e) {
|
||||
setErr(e instanceof Error ? e.message : String(e));
|
||||
}
|
||||
}, [t]);
|
||||
|
||||
const clickImage = useCallback(async (e: React.MouseEvent) => {
|
||||
if (!t || state?.holder !== "user" || !imgRef.current) return;
|
||||
const rect = imgRef.current.getBoundingClientRect();
|
||||
const x = Math.round(((e.clientX - rect.left) / rect.width) * 1280);
|
||||
const y = Math.round(((e.clientY - rect.top) / rect.height) * 800);
|
||||
try {
|
||||
await api(paths.taskBrowserInput(t.id), "POST", {
|
||||
type: "mouse_click", x, y,
|
||||
});
|
||||
} catch (err) {
|
||||
setErr(err instanceof Error ? err.message : String(err));
|
||||
}
|
||||
}, [t, state]);
|
||||
|
||||
const keyDown = useCallback(async (e: React.KeyboardEvent) => {
|
||||
if (!t || state?.holder !== "user") return;
|
||||
e.preventDefault();
|
||||
try {
|
||||
if (e.key.length === 1) {
|
||||
await api(paths.taskBrowserInput(t.id), "POST", {
|
||||
type: "type", text: e.key,
|
||||
});
|
||||
} else {
|
||||
await api(paths.taskBrowserInput(t.id), "POST", {
|
||||
type: "key", key: e.key,
|
||||
});
|
||||
}
|
||||
} catch (err) {
|
||||
setErr(err instanceof Error ? err.message : String(err));
|
||||
}
|
||||
}, [t, state]);
|
||||
|
||||
if (!t) return <div className="muted">点任务行查看详情</div>;
|
||||
const holder = state?.holder || "—";
|
||||
const active = state !== null;
|
||||
return (
|
||||
<div>
|
||||
<div style={{ display: "flex", gap: 8, flexWrap: "wrap", alignItems: "center" }}>
|
||||
<span>状态 <b>{status}</b></span>
|
||||
<span className="muted">会话 {active ? "活跃" : "未激活（轮次结束即消失）"}</span>
|
||||
<span className="muted">控制权 {holder}</span>
|
||||
{state?.url ? <span className="muted" title={state.url}>{state.url.slice(0, 80)}</span> : null}
|
||||
</div>
|
||||
<div style={{ display: "flex", gap: 8, flexWrap: "wrap", marginTop: 8 }}>
|
||||
{!watching
|
||||
? <button type="button" onClick={() => void startWatch()}>开始观看</button>
|
||||
: <button type="button" onClick={stopWatch}>停止观看</button>}
|
||||
<button type="button" onClick={() => void refreshState()}>刷新状态</button>
|
||||
{holder !== "user"
|
||||
? <button type="button" onClick={() => void takeover("user")} disabled={!active}>接管</button>
|
||||
: <button type="button" onClick={() => void takeover("agent")}>交还给 Agent</button>}
|
||||
</div>
|
||||
{err ? <div className="muted" style={{ marginTop: 8 }}>{err}</div> : null}
|
||||
{holder === "user" ? (
|
||||
<div className="muted" style={{ marginTop: 8 }}>
|
||||
你已接管：点画面即点击，键盘输入直达页面。交还后 Agent 续跑前会先快照确认你的改动。
|
||||
</div>
|
||||
) : null}
|
||||
<div style={{ marginTop: 8 }}>
|
||||
{img ? (
|
||||
<img
|
||||
ref={imgRef}
|
||||
src={img}
|
||||
alt="浏览器实时画面"
|
||||
tabIndex={0}
|
||||
onClick={(e) => void clickImage(e)}
|
||||
onKeyDown={(e) => void keyDown(e)}
|
||||
style={{
|
||||
width: "100%",
|
||||
maxWidth: 1280,
|
||||
cursor: holder === "user" ? "crosshair" : "default",
|
||||
border: "1px solid var(--border, #ccc)",
|
||||
}}
|
||||
/>
|
||||
) : (
|
||||
<div className="muted">
|
||||
{active
|
||||
? "点“开始观看”接入画面流（接管后点画面即点击）"
|
||||
: "浏览器会话未激活：任务本轮未开浏览器，或轮次已结束。"}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
Reference in new issue
Block a user