feat: 浏览器二期后端:远控服务 + ref 快照 + server 代理

- playwright_remote.py(新):MCP 进程内 Unix socket HTTP 服务,
  监听 worktree/.browser-remote.sock;GET /state、GET /stream
  (SSE 画面流)、POST /input、POST /control(接管/交还)。
  线程契约(实测):Playwright 同步调用只能在创建页面的线程,
  所以 HTTP 线程只读属性/读最新帧/入队,MCP 主循环用
  select 轮询(50ms) draining 输入并 pump screencast 帧
  (无 ack 则 CDP 首帧后停流)。
- 画面流走 CDP screencast 帧(学 chrome-devtools-mcp),不用
  定时截图;ffmpeg 录像暂不做。
- snapshot_refs/click_ref(学 playwright-mcp 的 ref 机制)+
  mouse_move/key_press/type_text;agent 工具在 holder=user 时
  直接报错,等用户交还。
- server.py:/tasks/{id}/browser/{state,stream,input,control}
  代理到 worktree socket,复用任务鉴权与 worktree 解析;
  会话不存在时 503(轮次结束即消失是设计)。
- 播种器:MCP 脚本是系统文件,版本变化时刷新老租户旧脚本
  (script_refreshed 打日志),否则远控只对新项目生效;
  新增 playwright_remote.py 进播种清单。
- 队列遮蔽修复:cloud/queue.py 会遮蔽 stdlib queue,改用
  SimpleQueue(deque+锁),仓库模式与 flat-copy 两路都通。
- 测试:新 20 项(remote 服务、server 代理、ref/输入/
  screencast fake),全量 1354 项通过(skipped 4)。
This commit is contained in:
host committed 2026-10-30 06:44:24 +08:00
1 parent 7bea8df512
commit 6f39ab3945
9 files changed
+1231 -13

No files matched your search

+186 -3
View File
@@ -147,6 +147,58 @@ _TOOLS = [
},
},
},
{
"name": "browser_snapshot_refs",
"description": (
"Accessibility-style snapshot: interactive elements "
"with stable refs (e.g. e3) plus tag, role and name. "
"Use browser_click_ref to click one; refs are valid "
"until the next snapshot_refs call."
),
"inputSchema": {"type": "object", "properties": {}},
},
{
"name": "browser_click_ref",
"description": (
"Click an element by its ref from browser_snapshot_refs. "
"More robust than a CSS selector across re-renders."
),
"inputSchema": {
"type": "object",
"properties": {"ref": {"type": "string"}},
"required": ["ref"],
},
},
{
"name": "browser_mouse_move",
"description": "Move the mouse to viewport coordinates (1280x800).",
"inputSchema": {
"type": "object",
"properties": {
"x": {"type": "number"},
"y": {"type": "number"},
},
"required": ["x", "y"],
},
},
{
"name": "browser_key_press",
"description": "Press a key (Playwright key name, e.g. Enter, Tab).",
"inputSchema": {
"type": "object",
"properties": {"key": {"type": "string"}},
"required": ["key"],
},
},
{
"name": "browser_type_text",
"description": "Type text at the current focus.",
"inputSchema": {
"type": "object",
"properties": {"text": {"type": "string"}},
"required": ["text"],
},
},
{
"name": "browser_screenshot",
"description": (
@@ -178,6 +230,7 @@ def _err(code, message):
_BROWSER = None
_REMOTE = None
def _get_browser(browser=None):
@@ -189,6 +242,25 @@ def _get_browser(browser=None):
return _BROWSER
def _get_remote(browser=None):
"""Remote control state, lazily created. Keyed by browser
identity so tests passing different fake browsers don't share
one holder flag."""
global _REMOTE
target = _get_browser(browser)
if _REMOTE is not None and _REMOTE.browser is target:
return _REMOTE
try:
from playwright_remote import RemoteState
except ImportError:
try:
from cloud.playwright_remote import RemoteState
except ImportError:
return None
_REMOTE = RemoteState(target)
return _REMOTE
def handle_request(request, browser=None):
"""Handle one JSON-RPC request dict. Returns a response dict or
None for notifications (no id). Tests pass a fake browser."""
@@ -227,6 +299,14 @@ def handle_request(request, browser=None):
name = params.get("name")
args = params.get("arguments") or {}
target = _get_browser(browser)
remote = _get_remote(browser)
if remote is not None and remote.get_holder() == "user":
return {"jsonrpc": "2.0", "id": req_id,
"result": {"content": [{"type": "text",
"text": "error: user holds "
"browser control; wait for "
"release"}],
"isError": True}}
try:
if name == "browser_navigate":
result = target.navigate(args.get("url"))
@@ -239,6 +319,16 @@ def handle_request(request, browser=None):
args.get("text"))
elif name == "browser_click_at":
result = target.click_at(args.get("x"), args.get("y"))
elif name == "browser_mouse_move":
result = target.mouse_move(args.get("x"), args.get("y"))
elif name == "browser_key_press":
result = target.key_press(args.get("key"))
elif name == "browser_type_text":
result = target.type_text(args.get("text"))
elif name == "browser_snapshot_refs":
result = target.snapshot_refs()
elif name == "browser_click_ref":
result = target.click_ref(args.get("ref"))
elif name == "browser_live_shot":
result = target.live_shot(
args.get("output_filename"))
@@ -268,9 +358,75 @@ def handle_request(request, browser=None):
"error": _err(-32601, "method not found: %r" % method)}
def _drain_remote(browser):
"""Main-thread only: run queued user inputs and pump screencast
frames (acks). Called on every loop iteration so remote input
lands even while no JSON-RPC request is in flight."""
remote = _get_remote(browser)
if remote is None:
return
try:
from playwright_remote import execute_input
except ImportError:
try:
from cloud.playwright_remote import execute_input
except ImportError:
return
while True:
try:
item = remote.inputs.get_nowait()
except Exception: # noqa: BLE001 - queue empty
break
if remote.get_holder() != "user":
continue
try:
execute_input(browser, item)
except Exception as exc: # noqa: BLE001 - input errors are data
print("playwright_mcp: remote input error: %s" % exc,
file=sys.stderr)
try:
frame = browser.pump_screencast()
except Exception: # noqa: BLE001 - screencast best-effort
frame = ""
if frame:
remote.set_frame(frame)
def main():
for line in sys.stdin:
line = line.strip()
import select
browser = _get_browser()
remote = _get_remote(browser)
if remote is not None:
try:
from playwright_remote import start_remote_server
except ImportError:
try:
from cloud.playwright_remote import start_remote_server
except ImportError:
start_remote_server = None
if start_remote_server is not None:
socket_path = os.environ.get(
"PLAYWRIGHT_REMOTE_SOCKET",
os.path.join(browser.cwd, ".browser-remote.sock"))
try:
browser.set_frame_sink(_frame_queue())
start_remote_server(remote, socket_path)
except Exception as exc: # noqa: BLE001 - remote best-effort
print("playwright_mcp: remote server failed: %s" % exc,
file=sys.stderr)
buffer = ""
while True:
_drain_remote(browser)
try:
readable, _, _ = select.select([sys.stdin], [], [], 0.05)
except Exception: # noqa: BLE001 - stdin gone
break
if not readable:
continue
chunk = sys.stdin.readline()
if not chunk:
break
line = chunk.strip()
if not line:
continue
try:
@@ -282,7 +438,7 @@ def main():
sys.stdout.flush()
continue
try:
response = handle_request(request)
response = handle_request(request, browser)
except Exception as exc: # noqa: BLE001 - server must not die
print("playwright_mcp: handler error: %s" % exc,
file=sys.stderr)
@@ -296,5 +452,32 @@ def main():
sys.stdout.flush()
def _frame_queue():
try:
from playwright_remote import SimpleQueue
except ImportError:
try:
from cloud.playwright_remote import SimpleQueue
except ImportError:
import collections
import threading
class SimpleQueue: # noqa: D101 - last-resort fallback
def __init__(self):
self._deque = collections.deque()
self._lock = threading.Lock()
def put(self, item):
with self._lock:
self._deque.append(item)
def get_nowait(self):
with self._lock:
if not self._deque:
raise IndexError("queue empty")
return self._deque.popleft()
return SimpleQueue()
if __name__ == "__main__":
main()