feat: 浏览器二期后端:远控服务 + ref 快照 + server 代理
- playwright_remote.py(新):MCP 进程内 Unix socket HTTP 服务,
监听 worktree/.browser-remote.sock;GET /state、GET /stream
(SSE 画面流)、POST /input、POST /control(接管/交还)。
线程契约(实测):Playwright 同步调用只能在创建页面的线程,
所以 HTTP 线程只读属性/读最新帧/入队,MCP 主循环用
select 轮询(50ms) draining 输入并 pump screencast 帧
(无 ack 则 CDP 首帧后停流)。
- 画面流走 CDP screencast 帧(学 chrome-devtools-mcp),不用
定时截图;ffmpeg 录像暂不做。
- snapshot_refs/click_ref(学 playwright-mcp 的 ref 机制)+
mouse_move/key_press/type_text;agent 工具在 holder=user 时
直接报错,等用户交还。
- server.py:/tasks/{id}/browser/{state,stream,input,control}
代理到 worktree socket,复用任务鉴权与 worktree 解析;
会话不存在时 503(轮次结束即消失是设计)。
- 播种器:MCP 脚本是系统文件,版本变化时刷新老租户旧脚本
(script_refreshed 打日志),否则远控只对新项目生效;
新增 playwright_remote.py 进播种清单。
- 队列遮蔽修复:cloud/queue.py 会遮蔽 stdlib queue,改用
SimpleQueue(deque+锁),仓库模式与 flat-copy 两路都通。
- 测试:新 20 项(remote 服务、server 代理、ref/输入/
screencast fake),全量 1354 项通过(skipped 4)。
This commit is contained in:
1 parent
7bea8df512
commit
6f39ab3945
9 files changed
+1231
-13
No files matched your search
+186
-3
@@ -147,6 +147,58 @@ _TOOLS = [
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
"name": "browser_snapshot_refs",
|
||||
"description": (
|
||||
"Accessibility-style snapshot: interactive elements "
|
||||
"with stable refs (e.g. e3) plus tag, role and name. "
|
||||
"Use browser_click_ref to click one; refs are valid "
|
||||
"until the next snapshot_refs call."
|
||||
),
|
||||
"inputSchema": {"type": "object", "properties": {}},
|
||||
},
|
||||
{
|
||||
"name": "browser_click_ref",
|
||||
"description": (
|
||||
"Click an element by its ref from browser_snapshot_refs. "
|
||||
"More robust than a CSS selector across re-renders."
|
||||
),
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"properties": {"ref": {"type": "string"}},
|
||||
"required": ["ref"],
|
||||
},
|
||||
},
|
||||
{
|
||||
"name": "browser_mouse_move",
|
||||
"description": "Move the mouse to viewport coordinates (1280x800).",
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"x": {"type": "number"},
|
||||
"y": {"type": "number"},
|
||||
},
|
||||
"required": ["x", "y"],
|
||||
},
|
||||
},
|
||||
{
|
||||
"name": "browser_key_press",
|
||||
"description": "Press a key (Playwright key name, e.g. Enter, Tab).",
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"properties": {"key": {"type": "string"}},
|
||||
"required": ["key"],
|
||||
},
|
||||
},
|
||||
{
|
||||
"name": "browser_type_text",
|
||||
"description": "Type text at the current focus.",
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"properties": {"text": {"type": "string"}},
|
||||
"required": ["text"],
|
||||
},
|
||||
},
|
||||
{
|
||||
"name": "browser_screenshot",
|
||||
"description": (
|
||||
@@ -178,6 +230,7 @@ def _err(code, message):
|
||||
|
||||
|
||||
_BROWSER = None
|
||||
_REMOTE = None
|
||||
|
||||
|
||||
def _get_browser(browser=None):
|
||||
@@ -189,6 +242,25 @@ def _get_browser(browser=None):
|
||||
return _BROWSER
|
||||
|
||||
|
||||
def _get_remote(browser=None):
|
||||
"""Remote control state, lazily created. Keyed by browser
|
||||
identity so tests passing different fake browsers don't share
|
||||
one holder flag."""
|
||||
global _REMOTE
|
||||
target = _get_browser(browser)
|
||||
if _REMOTE is not None and _REMOTE.browser is target:
|
||||
return _REMOTE
|
||||
try:
|
||||
from playwright_remote import RemoteState
|
||||
except ImportError:
|
||||
try:
|
||||
from cloud.playwright_remote import RemoteState
|
||||
except ImportError:
|
||||
return None
|
||||
_REMOTE = RemoteState(target)
|
||||
return _REMOTE
|
||||
|
||||
|
||||
def handle_request(request, browser=None):
|
||||
"""Handle one JSON-RPC request dict. Returns a response dict or
|
||||
None for notifications (no id). Tests pass a fake browser."""
|
||||
@@ -227,6 +299,14 @@ def handle_request(request, browser=None):
|
||||
name = params.get("name")
|
||||
args = params.get("arguments") or {}
|
||||
target = _get_browser(browser)
|
||||
remote = _get_remote(browser)
|
||||
if remote is not None and remote.get_holder() == "user":
|
||||
return {"jsonrpc": "2.0", "id": req_id,
|
||||
"result": {"content": [{"type": "text",
|
||||
"text": "error: user holds "
|
||||
"browser control; wait for "
|
||||
"release"}],
|
||||
"isError": True}}
|
||||
try:
|
||||
if name == "browser_navigate":
|
||||
result = target.navigate(args.get("url"))
|
||||
@@ -239,6 +319,16 @@ def handle_request(request, browser=None):
|
||||
args.get("text"))
|
||||
elif name == "browser_click_at":
|
||||
result = target.click_at(args.get("x"), args.get("y"))
|
||||
elif name == "browser_mouse_move":
|
||||
result = target.mouse_move(args.get("x"), args.get("y"))
|
||||
elif name == "browser_key_press":
|
||||
result = target.key_press(args.get("key"))
|
||||
elif name == "browser_type_text":
|
||||
result = target.type_text(args.get("text"))
|
||||
elif name == "browser_snapshot_refs":
|
||||
result = target.snapshot_refs()
|
||||
elif name == "browser_click_ref":
|
||||
result = target.click_ref(args.get("ref"))
|
||||
elif name == "browser_live_shot":
|
||||
result = target.live_shot(
|
||||
args.get("output_filename"))
|
||||
@@ -268,9 +358,75 @@ def handle_request(request, browser=None):
|
||||
"error": _err(-32601, "method not found: %r" % method)}
|
||||
|
||||
|
||||
def _drain_remote(browser):
|
||||
"""Main-thread only: run queued user inputs and pump screencast
|
||||
frames (acks). Called on every loop iteration so remote input
|
||||
lands even while no JSON-RPC request is in flight."""
|
||||
remote = _get_remote(browser)
|
||||
if remote is None:
|
||||
return
|
||||
try:
|
||||
from playwright_remote import execute_input
|
||||
except ImportError:
|
||||
try:
|
||||
from cloud.playwright_remote import execute_input
|
||||
except ImportError:
|
||||
return
|
||||
while True:
|
||||
try:
|
||||
item = remote.inputs.get_nowait()
|
||||
except Exception: # noqa: BLE001 - queue empty
|
||||
break
|
||||
if remote.get_holder() != "user":
|
||||
continue
|
||||
try:
|
||||
execute_input(browser, item)
|
||||
except Exception as exc: # noqa: BLE001 - input errors are data
|
||||
print("playwright_mcp: remote input error: %s" % exc,
|
||||
file=sys.stderr)
|
||||
try:
|
||||
frame = browser.pump_screencast()
|
||||
except Exception: # noqa: BLE001 - screencast best-effort
|
||||
frame = ""
|
||||
if frame:
|
||||
remote.set_frame(frame)
|
||||
|
||||
|
||||
def main():
|
||||
for line in sys.stdin:
|
||||
line = line.strip()
|
||||
import select
|
||||
browser = _get_browser()
|
||||
remote = _get_remote(browser)
|
||||
if remote is not None:
|
||||
try:
|
||||
from playwright_remote import start_remote_server
|
||||
except ImportError:
|
||||
try:
|
||||
from cloud.playwright_remote import start_remote_server
|
||||
except ImportError:
|
||||
start_remote_server = None
|
||||
if start_remote_server is not None:
|
||||
socket_path = os.environ.get(
|
||||
"PLAYWRIGHT_REMOTE_SOCKET",
|
||||
os.path.join(browser.cwd, ".browser-remote.sock"))
|
||||
try:
|
||||
browser.set_frame_sink(_frame_queue())
|
||||
start_remote_server(remote, socket_path)
|
||||
except Exception as exc: # noqa: BLE001 - remote best-effort
|
||||
print("playwright_mcp: remote server failed: %s" % exc,
|
||||
file=sys.stderr)
|
||||
buffer = ""
|
||||
while True:
|
||||
_drain_remote(browser)
|
||||
try:
|
||||
readable, _, _ = select.select([sys.stdin], [], [], 0.05)
|
||||
except Exception: # noqa: BLE001 - stdin gone
|
||||
break
|
||||
if not readable:
|
||||
continue
|
||||
chunk = sys.stdin.readline()
|
||||
if not chunk:
|
||||
break
|
||||
line = chunk.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
@@ -282,7 +438,7 @@ def main():
|
||||
sys.stdout.flush()
|
||||
continue
|
||||
try:
|
||||
response = handle_request(request)
|
||||
response = handle_request(request, browser)
|
||||
except Exception as exc: # noqa: BLE001 - server must not die
|
||||
print("playwright_mcp: handler error: %s" % exc,
|
||||
file=sys.stderr)
|
||||
@@ -296,5 +452,32 @@ def main():
|
||||
sys.stdout.flush()
|
||||
|
||||
|
||||
def _frame_queue():
|
||||
try:
|
||||
from playwright_remote import SimpleQueue
|
||||
except ImportError:
|
||||
try:
|
||||
from cloud.playwright_remote import SimpleQueue
|
||||
except ImportError:
|
||||
import collections
|
||||
import threading
|
||||
|
||||
class SimpleQueue: # noqa: D101 - last-resort fallback
|
||||
def __init__(self):
|
||||
self._deque = collections.deque()
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def put(self, item):
|
||||
with self._lock:
|
||||
self._deque.append(item)
|
||||
|
||||
def get_nowait(self):
|
||||
with self._lock:
|
||||
if not self._deque:
|
||||
raise IndexError("queue empty")
|
||||
return self._deque.popleft()
|
||||
return SimpleQueue()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in new issue
Block a user