feat: 浏览器二期后端:远控服务 + ref 快照 + server 代理
- playwright_remote.py(新):MCP 进程内 Unix socket HTTP 服务,
监听 worktree/.browser-remote.sock;GET /state、GET /stream
(SSE 画面流)、POST /input、POST /control(接管/交还)。
线程契约(实测):Playwright 同步调用只能在创建页面的线程,
所以 HTTP 线程只读属性/读最新帧/入队,MCP 主循环用
select 轮询(50ms) draining 输入并 pump screencast 帧
(无 ack 则 CDP 首帧后停流)。
- 画面流走 CDP screencast 帧(学 chrome-devtools-mcp),不用
定时截图;ffmpeg 录像暂不做。
- snapshot_refs/click_ref(学 playwright-mcp 的 ref 机制)+
mouse_move/key_press/type_text;agent 工具在 holder=user 时
直接报错,等用户交还。
- server.py:/tasks/{id}/browser/{state,stream,input,control}
代理到 worktree socket,复用任务鉴权与 worktree 解析;
会话不存在时 503(轮次结束即消失是设计)。
- 播种器:MCP 脚本是系统文件,版本变化时刷新老租户旧脚本
(script_refreshed 打日志),否则远控只对新项目生效;
新增 playwright_remote.py 进播种清单。
- 队列遮蔽修复:cloud/queue.py 会遮蔽 stdlib queue,改用
SimpleQueue(deque+锁),仓库模式与 flat-copy 两路都通。
- 测试:新 20 项(remote 服务、server 代理、ref/输入/
screencast fake),全量 1354 项通过(skipped 4)。
This commit is contained in:
1 parent
7bea8df512
commit
6f39ab3945
9 files changed
+1231
-13
No files matched your search
@@ -25,6 +25,13 @@ class _FakeLocator:
|
||||
def inner_text(self, timeout=None):
|
||||
return self._text
|
||||
|
||||
@property
|
||||
def first(self):
|
||||
return self
|
||||
|
||||
def click(self, timeout=None):
|
||||
pass
|
||||
|
||||
|
||||
class _FakeMouse:
|
||||
def __init__(self, page):
|
||||
@@ -33,6 +40,40 @@ class _FakeMouse:
|
||||
def click(self, x, y):
|
||||
self._page.calls.append(("mouse_click", x, y))
|
||||
|
||||
def move(self, x, y):
|
||||
self._page.calls.append(("mouse_move", x, y))
|
||||
|
||||
|
||||
class _FakeKeyboard:
|
||||
def __init__(self, page):
|
||||
self._page = page
|
||||
|
||||
def press(self, key):
|
||||
self._page.calls.append(("key_press", key))
|
||||
|
||||
def type(self, text):
|
||||
self._page.calls.append(("key_type", text))
|
||||
|
||||
|
||||
class _FakeCDP:
|
||||
def __init__(self):
|
||||
self.sent = []
|
||||
self.handlers = {}
|
||||
|
||||
def on(self, event, handler):
|
||||
self.handlers[event] = handler
|
||||
|
||||
def send(self, method, params=None):
|
||||
self.sent.append((method, params or {}))
|
||||
|
||||
|
||||
class _FakeContext:
|
||||
def __init__(self, cdp):
|
||||
self.cdp = cdp
|
||||
|
||||
def new_cdp_session(self, page):
|
||||
return self.cdp
|
||||
|
||||
|
||||
class _FakePage:
|
||||
def __init__(self):
|
||||
@@ -40,6 +81,9 @@ class _FakePage:
|
||||
self.calls = []
|
||||
self.shot_path = None
|
||||
self.mouse = _FakeMouse(self)
|
||||
self.keyboard = _FakeKeyboard(self)
|
||||
self.cdp = _FakeCDP()
|
||||
self.context = _FakeContext(self.cdp)
|
||||
|
||||
def title(self):
|
||||
return "Example Domain"
|
||||
@@ -60,6 +104,11 @@ class _FakePage:
|
||||
|
||||
def evaluate(self, expression):
|
||||
self.calls.append(("evaluate", expression))
|
||||
if "querySelectorAll" in expression:
|
||||
return [{"tag": "button", "role": "", "name": "GO",
|
||||
"path": "button"},
|
||||
{"tag": "a", "role": "link", "name": "Home",
|
||||
"path": "a"}]
|
||||
return 42
|
||||
|
||||
def screenshot(self, path=None):
|
||||
@@ -166,6 +215,50 @@ class BrowserTest(unittest.TestCase):
|
||||
with self.assertRaises(BrowserError):
|
||||
self.browser.click_at("left", 10)
|
||||
|
||||
def test_remote_input_methods(self):
|
||||
self.browser.navigate("https://example.com")
|
||||
out = self.browser.mouse_move(10, 20)
|
||||
self.assertTrue(out["ok"])
|
||||
self.assertIn(("mouse_move", 10, 20), self.page.calls)
|
||||
out = self.browser.key_press("Enter")
|
||||
self.assertTrue(out["ok"])
|
||||
self.assertIn(("key_press", "Enter"), self.page.calls)
|
||||
out = self.browser.type_text("hi")
|
||||
self.assertTrue(out["ok"])
|
||||
self.assertIn(("key_type", "hi"), self.page.calls)
|
||||
with self.assertRaises(BrowserError):
|
||||
self.browser.mouse_move(9999, 10)
|
||||
|
||||
def test_snapshot_refs_and_click_ref(self):
|
||||
self.browser.navigate("https://example.com")
|
||||
out = self.browser.snapshot_refs()
|
||||
self.assertEqual(out["count"], 2)
|
||||
self.assertEqual(out["elements"][0]["ref"], "e0")
|
||||
self.assertEqual(out["elements"][0]["name"], "GO")
|
||||
clicked = self.browser.click_ref("e0")
|
||||
self.assertTrue(clicked["ok"])
|
||||
self.assertEqual(clicked["ref"], "e0")
|
||||
self.assertIn(("locator", "button"), self.page.calls)
|
||||
with self.assertRaises(BrowserError):
|
||||
self.browser.click_ref("e99")
|
||||
|
||||
def test_screencast_attach_and_pump(self):
|
||||
from playwright_remote import SimpleQueue
|
||||
sink = SimpleQueue()
|
||||
self.browser.set_frame_sink(sink)
|
||||
self.browser.navigate("https://example.com")
|
||||
self.assertIn(("Page.startScreencast",
|
||||
{"format": "jpeg", "quality": 60,
|
||||
"maxWidth": 1280, "maxHeight": 800,
|
||||
"everyNthFrame": 2}),
|
||||
[(m, p) for m, p in self.page.cdp.sent])
|
||||
handler = self.page.cdp.handlers["Page.screencastFrame"]
|
||||
handler({"sessionId": "s1", "data": "FRAMEDATA"})
|
||||
frame = self.browser.pump_screencast()
|
||||
self.assertEqual(frame, "FRAMEDATA")
|
||||
self.assertIn(("Page.screencastFrameAck", {"sessionId": "s1"}),
|
||||
[(m, p) for m, p in self.page.cdp.sent])
|
||||
|
||||
def test_live_shot_writes_fixed_file_and_status(self):
|
||||
self.browser.navigate("https://example.com")
|
||||
out = self.browser.live_shot()
|
||||
@@ -192,6 +285,22 @@ class FakeMCPBrowser:
|
||||
"path": "/w/browser-live.png", "bytes": 456,
|
||||
"url": "https://x/", "title": "T", "timestamp": 1}
|
||||
|
||||
def mouse_move(self, x, y):
|
||||
return {"ok": True, "x": x, "y": y}
|
||||
|
||||
def key_press(self, key):
|
||||
return {"ok": True, "key": key}
|
||||
|
||||
def type_text(self, text):
|
||||
return {"ok": True}
|
||||
|
||||
def snapshot_refs(self):
|
||||
return {"elements": [{"ref": "e0", "tag": "button",
|
||||
"role": "", "name": "GO"}], "count": 1}
|
||||
|
||||
def click_ref(self, ref):
|
||||
return {"ok": True, "ref": ref}
|
||||
|
||||
def snapshot(self):
|
||||
return {"url": "https://x/", "title": "T", "text": "hi",
|
||||
"truncated": False}
|
||||
@@ -223,6 +332,32 @@ class MCPTest(unittest.TestCase):
|
||||
self.assertIn("https://example.com", text)
|
||||
self.assertFalse(resp["result"]["isError"])
|
||||
|
||||
def test_holder_user_blocks_agent_tools(self):
|
||||
import playwright_mcp
|
||||
fake = FakeMCPBrowser()
|
||||
remote = playwright_mcp._get_remote(fake)
|
||||
remote.set_holder("user")
|
||||
try:
|
||||
resp = handle_request(
|
||||
{"jsonrpc": "2.0", "id": 3, "method": "tools/call",
|
||||
"params": {"name": "browser_navigate",
|
||||
"arguments": {"url": "https://example.com"}}},
|
||||
browser=fake)
|
||||
self.assertTrue(resp["result"]["isError"])
|
||||
self.assertIn("holds browser control",
|
||||
resp["result"]["content"][0]["text"])
|
||||
finally:
|
||||
remote.set_holder("agent")
|
||||
|
||||
def test_new_tools_listed(self):
|
||||
listed = handle_request({"jsonrpc": "2.0", "id": 4,
|
||||
"method": "tools/list"})
|
||||
names = [t["name"] for t in listed["result"]["tools"]]
|
||||
for name in ("browser_snapshot_refs", "browser_click_ref",
|
||||
"browser_mouse_move", "browser_key_press",
|
||||
"browser_type_text"):
|
||||
self.assertIn(name, names)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,156 @@
|
||||
"""Tests for the Playwright remote-control service (fake only).
|
||||
|
||||
Covers: holder transitions + input gating, the Unix-socket HTTP
|
||||
endpoints (state/control/input/stream), and the main-thread
|
||||
input execution path. No real Chromium: the CDP/screencast path
|
||||
is exercised through fakes, real-browser proof lives in the
|
||||
production verification task.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import socket
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import unittest
|
||||
import urllib.request
|
||||
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
|
||||
|
||||
from playwright_remote import (RemoteState, execute_input, # noqa: E402
|
||||
start_remote_server)
|
||||
|
||||
|
||||
class _StubBrowser:
|
||||
def __init__(self):
|
||||
self.calls = []
|
||||
|
||||
def cached_info(self):
|
||||
return {"url": "https://example.com", "title": "T",
|
||||
"started": True}
|
||||
|
||||
def click_at(self, x, y):
|
||||
self.calls.append(("click_at", x, y))
|
||||
return {"ok": True}
|
||||
|
||||
def mouse_move(self, x, y):
|
||||
self.calls.append(("mouse_move", x, y))
|
||||
return {"ok": True}
|
||||
|
||||
def key_press(self, key):
|
||||
self.calls.append(("key_press", key))
|
||||
return {"ok": True}
|
||||
|
||||
def type_text(self, text):
|
||||
self.calls.append(("type_text", text))
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
class RemoteStateTest(unittest.TestCase):
|
||||
def test_holder_and_input_gating(self):
|
||||
state = RemoteState(_StubBrowser())
|
||||
self.assertEqual(state.get_holder(), "agent")
|
||||
with self.assertRaises(PermissionError):
|
||||
state.enqueue_input({"type": "mouse_click", "x": 1, "y": 2})
|
||||
state.set_holder("user")
|
||||
state.enqueue_input({"type": "mouse_click", "x": 10, "y": 20})
|
||||
item = state.inputs.get_nowait()
|
||||
execute_input(state.browser, item)
|
||||
self.assertIn(("click_at", 10, 20), state.browser.calls)
|
||||
with self.assertRaises(ValueError):
|
||||
state.set_holder("nobody")
|
||||
with self.assertRaises(ValueError):
|
||||
execute_input(state.browser, {"type": "teleport"})
|
||||
|
||||
def test_state_dict(self):
|
||||
state = RemoteState(_StubBrowser())
|
||||
state.set_frame("abc")
|
||||
d = state.state_dict()
|
||||
self.assertEqual(d["holder"], "agent")
|
||||
self.assertTrue(d["streaming"])
|
||||
self.assertEqual(d["url"], "https://example.com")
|
||||
|
||||
|
||||
class HttpServiceTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self._tmp = tempfile.TemporaryDirectory()
|
||||
self.browser = _StubBrowser()
|
||||
self.state = RemoteState(self.browser)
|
||||
self.sock = os.path.join(self._tmp.name, "remote.sock")
|
||||
self.server = start_remote_server(self.state, self.sock)
|
||||
|
||||
def tearDown(self):
|
||||
self.server.shutdown()
|
||||
self.server.server_close()
|
||||
self._tmp.cleanup()
|
||||
|
||||
def _unix_get(self, path):
|
||||
import http.client
|
||||
|
||||
class UnixConn(http.client.HTTPConnection):
|
||||
def connect(self):
|
||||
self.sock = socket.socket(socket.AF_UNIX,
|
||||
socket.SOCK_STREAM)
|
||||
self.sock.connect(self._sock_path)
|
||||
|
||||
conn = UnixConn("localhost", timeout=5)
|
||||
conn._sock_path = self.sock
|
||||
conn.request("GET", path)
|
||||
resp = conn.getresponse()
|
||||
return resp.status, json.loads(resp.read().decode("utf-8"))
|
||||
|
||||
def _unix_post(self, path, obj):
|
||||
import http.client
|
||||
|
||||
class UnixConn(http.client.HTTPConnection):
|
||||
def connect(self):
|
||||
self.sock = socket.socket(socket.AF_UNIX,
|
||||
socket.SOCK_STREAM)
|
||||
self.sock.connect(self._sock_path)
|
||||
|
||||
conn = UnixConn("localhost", timeout=5)
|
||||
conn._sock_path = self.sock
|
||||
body = json.dumps(obj).encode("utf-8")
|
||||
conn.request("POST", path, body=body,
|
||||
headers={"Content-Type": "application/json"})
|
||||
resp = conn.getresponse()
|
||||
return resp.status, json.loads(resp.read().decode("utf-8"))
|
||||
|
||||
def test_state_control_input_flow(self):
|
||||
status, body = self._unix_get("/state")
|
||||
self.assertEqual(status, 200)
|
||||
self.assertEqual(body["holder"], "agent")
|
||||
status, body = self._unix_post(
|
||||
"/input", {"type": "mouse_click", "x": 1, "y": 2})
|
||||
self.assertEqual(status, 409)
|
||||
status, body = self._unix_post("/control", {"holder": "user"})
|
||||
self.assertEqual((status, body["holder"]), (200, "user"))
|
||||
status, body = self._unix_post(
|
||||
"/input", {"type": "key", "key": "Enter"})
|
||||
self.assertEqual((status, body["queued"]), (200, True))
|
||||
self.assertEqual(self.state.inputs.get_nowait()["key"], "Enter")
|
||||
status, body = self._unix_post("/control", {"holder": "agent"})
|
||||
self.assertEqual(body["holder"], "agent")
|
||||
|
||||
def test_stream_emits_latest_frame(self):
|
||||
self.state.set_frame("FRAMEBYTES")
|
||||
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
||||
sock.settimeout(5)
|
||||
sock.connect(self.sock)
|
||||
sock.sendall(b"GET /stream HTTP/1.1\r\nHost: x\r\n\r\n")
|
||||
buf = b""
|
||||
while b"\n\n" not in buf:
|
||||
buf += sock.recv(4096)
|
||||
head, _, rest = buf.partition(b"\n\n")
|
||||
self.assertIn(b"text/event-stream", head)
|
||||
first = rest
|
||||
while b"\n\n" not in first:
|
||||
first += sock.recv(4096)
|
||||
payload = json.loads(first.split(b"data: ", 1)[1].split(b"\n\n")[0])
|
||||
self.assertEqual(payload["image"], "FRAMEBYTES")
|
||||
sock.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,140 @@
|
||||
"""Browser proxy endpoints: /tasks/{id}/browser/{state,input,control}.
|
||||
|
||||
The control plane forwards to the MCP remote service over the Unix
|
||||
socket in the task worktree. Here the socket server is the real
|
||||
playwright_remote module with a stub browser; no Chromium, no
|
||||
worker, no model.
|
||||
"""
|
||||
|
||||
import http.client
|
||||
import json
|
||||
import os
|
||||
import sqlite3
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
from cloud.auth import AuthStore
|
||||
from cloud.playwright_remote import RemoteState, start_remote_server
|
||||
from cloud.server import create_server
|
||||
except ImportError:
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||||
from cloud.auth import AuthStore
|
||||
from cloud.playwright_remote import RemoteState, start_remote_server
|
||||
from cloud.server import create_server
|
||||
|
||||
|
||||
class _StubBrowser:
|
||||
def cached_info(self):
|
||||
return {"url": "https://example.com", "title": "T",
|
||||
"started": True}
|
||||
|
||||
|
||||
class BrowserProxyTestCase(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.db_path = os.path.join(self.tmp.name, "control.db")
|
||||
self.auth = AuthStore(self.db_path)
|
||||
self.owner_id, self.owner_key = self.auth.create_user("owner")
|
||||
self.outsider_id, self.outsider_key = self.auth.create_user("o")
|
||||
self.server = create_server(self.db_path, "127.0.0.1", 0)
|
||||
self.worktrees = os.path.join(self.tmp.name, "host-worktrees")
|
||||
os.makedirs(self.worktrees)
|
||||
self.server.host_worktrees_dir = self.worktrees
|
||||
self.port = self.server.server_address[1]
|
||||
self.thread = threading.Thread(
|
||||
target=self.server.serve_forever, daemon=True)
|
||||
self.thread.start()
|
||||
status, self.project = self._req(
|
||||
"POST", "/projects", {"name": "demo"}, self.owner_key)
|
||||
self.assertEqual(status, 201)
|
||||
self._task("task-b1")
|
||||
wt = os.path.join(self.worktrees, "task-b1")
|
||||
os.makedirs(wt)
|
||||
self.remote = RemoteState(_StubBrowser())
|
||||
self.sock_server = start_remote_server(
|
||||
self.remote, os.path.join(wt, ".browser-remote.sock"))
|
||||
self._task("task-nosock")
|
||||
os.makedirs(os.path.join(self.worktrees, "task-nosock"))
|
||||
|
||||
def tearDown(self):
|
||||
self.sock_server.shutdown()
|
||||
self.sock_server.server_close()
|
||||
self.server.shutdown()
|
||||
self.server.server_close()
|
||||
self.thread.join(timeout=5)
|
||||
try:
|
||||
self.auth.conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
self.tmp.cleanup()
|
||||
|
||||
def _task(self, task_id):
|
||||
conn = sqlite3.connect(self.db_path)
|
||||
try:
|
||||
conn.execute(
|
||||
"INSERT INTO tasks(id, project_id, prompt, status)"
|
||||
" VALUES(?, ?, ?, ?)",
|
||||
(task_id, self.project["id"], "do it", "running"))
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def _req(self, method, path, body=None, key=None):
|
||||
conn = http.client.HTTPConnection("127.0.0.1", self.port, timeout=10)
|
||||
hdrs = {}
|
||||
payload = None
|
||||
if body is not None:
|
||||
payload = json.dumps(body).encode("utf-8")
|
||||
hdrs["Content-Type"] = "application/json"
|
||||
if key is not None:
|
||||
hdrs["X-API-Key"] = key
|
||||
try:
|
||||
conn.request(method, path, body=payload, headers=hdrs)
|
||||
resp = conn.getresponse()
|
||||
data = resp.read()
|
||||
finally:
|
||||
conn.close()
|
||||
try:
|
||||
parsed = json.loads(data.decode("utf-8")) if data else None
|
||||
except ValueError:
|
||||
parsed = None
|
||||
return resp.status, parsed
|
||||
|
||||
def test_state_input_control_flow(self):
|
||||
status, body = self._req(
|
||||
"GET", "/tasks/task-b1/browser/state", key=self.owner_key)
|
||||
self.assertEqual(status, 200, body)
|
||||
self.assertEqual(body["holder"], "agent")
|
||||
self.assertEqual(body["url"], "https://example.com")
|
||||
status, body = self._req(
|
||||
"POST", "/tasks/task-b1/browser/input",
|
||||
{"type": "mouse_click", "x": 1, "y": 2}, self.owner_key)
|
||||
self.assertEqual(status, 409, body)
|
||||
status, body = self._req(
|
||||
"POST", "/tasks/task-b1/browser/control",
|
||||
{"holder": "user"}, self.owner_key)
|
||||
self.assertEqual((status, body["holder"]), (200, "user"))
|
||||
status, body = self._req(
|
||||
"POST", "/tasks/task-b1/browser/input",
|
||||
{"type": "key", "key": "Enter"}, self.owner_key)
|
||||
self.assertEqual((status, body["queued"]), (200, True))
|
||||
status, body = self._req(
|
||||
"POST", "/tasks/task-b1/browser/control",
|
||||
{"holder": "agent"}, self.owner_key)
|
||||
self.assertEqual(body["holder"], "agent")
|
||||
|
||||
def test_forbidden_and_no_session(self):
|
||||
status, _ = self._req(
|
||||
"GET", "/tasks/task-b1/browser/state", key=self.outsider_key)
|
||||
self.assertEqual(status, 403)
|
||||
status, body = self._req(
|
||||
"GET", "/tasks/task-nosock/browser/state", key=self.owner_key)
|
||||
self.assertEqual(status, 503, body)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user