feat: 浏览器二期后端:远控服务 + ref 快照 + server 代理

- playwright_remote.py(新):MCP 进程内 Unix socket HTTP 服务,
  监听 worktree/.browser-remote.sock;GET /state、GET /stream
  (SSE 画面流)、POST /input、POST /control(接管/交还)。
  线程契约(实测):Playwright 同步调用只能在创建页面的线程,
  所以 HTTP 线程只读属性/读最新帧/入队,MCP 主循环用
  select 轮询(50ms) draining 输入并 pump screencast 帧
  (无 ack 则 CDP 首帧后停流)。
- 画面流走 CDP screencast 帧(学 chrome-devtools-mcp),不用
  定时截图;ffmpeg 录像暂不做。
- snapshot_refs/click_ref(学 playwright-mcp 的 ref 机制)+
  mouse_move/key_press/type_text;agent 工具在 holder=user 时
  直接报错,等用户交还。
- server.py:/tasks/{id}/browser/{state,stream,input,control}
  代理到 worktree socket,复用任务鉴权与 worktree 解析;
  会话不存在时 503(轮次结束即消失是设计)。
- 播种器:MCP 脚本是系统文件,版本变化时刷新老租户旧脚本
  (script_refreshed 打日志),否则远控只对新项目生效;
  新增 playwright_remote.py 进播种清单。
- 队列遮蔽修复:cloud/queue.py 会遮蔽 stdlib queue,改用
  SimpleQueue(deque+锁),仓库模式与 flat-copy 两路都通。
- 测试:新 20 项(remote 服务、server 代理、ref/输入/
  screencast fake),全量 1354 项通过(skipped 4)。
This commit is contained in:
host committed 2026-10-30 06:44:24 +08:00
1 parent 7bea8df512
commit 6f39ab3945
9 files changed
+1231 -13

No files matched your search

+220 -8
View File
@@ -61,6 +61,16 @@ class Browser:
self._browser = None
self._page = None
self._current_url = ""
self._last_title = ""
# ref snapshot state (playwright-mcp style): ref -> css path
self._ref_paths = {}
# screencast sink: a queue.Queue set by the remote service.
# The CDP frame callback only enqueues; the MCP main thread
# drains it (pump_screencast) because Playwright sync calls
# must run on the thread that created the page.
self._frame_sink = None
self._cdp = None
self._cdp_page = None
def _ensure(self):
if self._page is not None:
@@ -92,8 +102,15 @@ class Browser:
title = self._page.title()
except Exception: # noqa: BLE001 - title is best-effort
title = ""
self._last_title = title
return {"url": self._current_url, "title": title}
def cached_info(self):
"""Plain-attribute state, safe to read from any thread
(the remote HTTP thread must never call page.title())."""
return {"url": self._current_url, "title": self._last_title,
"started": self._page is not None}
def navigate(self, url):
if not url or not str(url).strip():
raise BrowserError("url is required")
@@ -102,6 +119,7 @@ class Browser:
self._page.goto(str(url), timeout=NAV_TIMEOUT_MS,
wait_until="domcontentloaded")
self._current_url = self._page.url
self._start_screencast()
except Exception as exc: # noqa: BLE001
raise BrowserError("navigate failed: %s" % exc)
result = self._info()
@@ -148,6 +166,185 @@ class Browser:
result["ok"] = True
return result
def mouse_move(self, x, y):
try:
fx = float(x)
fy = float(y)
except (TypeError, ValueError):
raise BrowserError("mouse_move needs numeric x/y, got %r/%r"
% (x, y))
if not (0 <= fx <= 1280 and 0 <= fy <= 800):
raise BrowserError(
"mouse_move out of viewport 1280x800: %s,%s" % (x, y))
self._ensure()
try:
self._page.mouse.move(fx, fy)
except Exception as exc: # noqa: BLE001
raise BrowserError("mouse_move failed: %s" % exc)
result = self._info()
result["ok"] = True
result["x"] = fx
result["y"] = fy
return result
def key_press(self, key):
if not key or not str(key).strip():
raise BrowserError("key is required")
self._ensure()
try:
self._page.keyboard.press(str(key))
except Exception as exc: # noqa: BLE001
raise BrowserError("key_press failed: %s" % exc)
result = self._info()
result["ok"] = True
result["key"] = str(key)
return result
def type_text(self, text):
if text is None:
raise BrowserError("text is required")
self._ensure()
try:
self._page.keyboard.type(str(text))
except Exception as exc: # noqa: BLE001
raise BrowserError("type_text failed: %s" % exc)
result = self._info()
result["ok"] = True
return result
def snapshot_refs(self):
"""Accessibility-style snapshot with refs (playwright-mcp
style): interactive elements get a stable ref valid until
the next snapshot_refs call; click_ref(ref) clicks one.
The ref maps to a computed CSS path, so it survives
re-renders better than an index into a stale list."""
self._ensure()
script = """
() => {
const path = (el) => {
const parts = [];
while (el && el.nodeType === 1 && el !== document.body) {
let tag = el.tagName.toLowerCase();
const parent = el.parentElement;
if (parent) {
const sibs = Array.from(parent.children).filter(
c => c.tagName === el.tagName);
if (sibs.length > 1)
tag += ':nth-of-type(' + (sibs.indexOf(el) + 1) + ')';
}
parts.unshift(tag);
el = parent;
}
return parts.join(' > ');
};
const out = [];
const els = document.querySelectorAll(
'a, button, input, select, textarea, [role="button"], ' +
'[role="link"], [role="textbox"], [role="checkbox"], ' +
'[role="radio"], [role="combobox"]');
els.forEach((el) => {
const r = el.getBoundingClientRect();
if (r.width === 0 && r.height === 0) return;
const name = (el.getAttribute('aria-label') ||
el.textContent || el.value || el.placeholder || '')
.trim().replace(/\\s+/g, ' ').slice(0, 80);
out.push({tag: el.tagName.toLowerCase(),
role: el.getAttribute('role') || '',
name: name, path: path(el)});
});
return out;
}
"""
try:
elements = self._page.evaluate(script)
except Exception as exc: # noqa: BLE001
raise BrowserError("snapshot_refs failed: %s" % exc)
self._ref_paths = {}
items = []
for i, el in enumerate(elements or []):
ref = "e%d" % i
self._ref_paths[ref] = el.get("path", "")
items.append({"ref": ref, "tag": el.get("tag", ""),
"role": el.get("role", ""),
"name": el.get("name", "")})
result = self._info()
result["elements"] = items
result["count"] = len(items)
return result
def click_ref(self, ref):
if not ref or ref not in self._ref_paths:
raise BrowserError(
"unknown ref %r: run browser_snapshot_refs first" % (ref,))
self._ensure()
path = self._ref_paths[ref]
try:
self._page.locator(path).first.click(timeout=10000)
self._current_url = self._page.url
except Exception as exc: # noqa: BLE001
raise BrowserError("click_ref failed: %s" % exc)
result = self._info()
result["ok"] = True
result["ref"] = ref
return result
def set_frame_sink(self, sink):
"""Remote service: frames arrive here via CDP callback."""
self._frame_sink = sink
if self._page is not None:
self._start_screencast()
def _start_screencast(self):
if self._frame_sink is None or self._page is None:
return
if self._cdp is not None and self._cdp_page is self._page:
return
try:
cdp = self._page.context.new_cdp_session(self._page)
def _on_frame(params):
try:
self._frame_sink.put(
(params.get("sessionId", ""),
params.get("data", "")))
except Exception: # noqa: BLE001 - callback must not die
pass
cdp.on("Page.screencastFrame", _on_frame)
cdp.send("Page.startScreencast",
{"format": "jpeg", "quality": 60,
"maxWidth": 1280, "maxHeight": 800,
"everyNthFrame": 2})
self._cdp = cdp
self._cdp_page = self._page
except Exception: # noqa: BLE001 - screencast is best-effort
self._cdp = None
self._cdp_page = None
def pump_screencast(self):
"""Main-thread only: store the latest frame and ack every
queued frame (no ack = CDP stops sending after the first).
Returns the latest frame data (base64 jpeg) or ''."""
if self._frame_sink is None:
return ""
latest = ""
while True:
try:
sid, data = self._frame_sink.get_nowait()
except Exception: # noqa: BLE001 - queue empty
break
if data:
latest = data
if self._cdp is not None:
try:
self._cdp.send("Page.screencastFrameAck",
{"sessionId": sid})
except Exception: # noqa: BLE001 - ack best-effort
pass
if latest:
self._latest_frame = latest
return getattr(self, "_latest_frame", "")
def click_at(self, x, y):
"""Click by viewport coordinates (CSS pixels, origin at the
top-left of the 1280x800 viewport). This is the tool the
@@ -226,6 +423,13 @@ class Browser:
def close(self):
errors = []
if self._cdp is not None:
try:
self._cdp.send("Page.stopScreencast")
except Exception: # noqa: BLE001 - best-effort
pass
self._cdp = None
self._cdp_page = None
for closer in (lambda: self._page and self._page.close(),
lambda: self._browser and self._browser.close(),
lambda: self._pw and self._pw.stop()):
@@ -245,11 +449,14 @@ def ensure_codex_home_playwright(codex_home_dir, command_args=None,
script_dir=None):
"""Seed playwright MCP files + config into a tenant codex-home.
Mirrors ``ensure_codex_home_imagegen``: copies
``playwright_browser.py`` and ``playwright_mcp.py`` only when
missing (a tenant-modified script is never silently
overwritten, it is reported in ``script_differs``), and
appends ``[mcp_servers.playwright]`` only when absent. No
MCP scripts are SYSTEM files, not tenant data: a tenant never
edits them (user files live in the worktree, not codex-home),
so unlike the old never-overwrite rule, a differing script is
REFRESHED to the deployed version and reported in
``script_refreshed``. Without this, every MCP feature ships
dead for existing projects: they keep the first-seeded script
forever and only brand-new projects get the update.
Appends ``[mcp_servers.playwright]`` only when absent. No
secrets live here -- proxy vars arrive via
``ensure_playwright_mcp_env`` from worker proxy_env, and the
browsers path is a fixed image path.
@@ -261,7 +468,9 @@ def ensure_codex_home_playwright(codex_home_dir, command_args=None,
here = os.path.abspath(script_dir or os.path.dirname(__file__))
script_changed = False
script_differs = []
for name in ("playwright_browser.py", "playwright_mcp.py"):
script_refreshed = []
for name in ("playwright_browser.py", "playwright_mcp.py",
"playwright_remote.py"):
src = os.path.join(here, name)
dst = os.path.join(home, name)
if not os.path.exists(src):
@@ -275,7 +484,9 @@ def ensure_codex_home_playwright(codex_home_dir, command_args=None,
with open(dst, "rb") as fh:
dst_bytes = fh.read()
if src_bytes != dst_bytes:
script_differs.append(name)
shutil.copyfile(src, dst)
script_refreshed.append(name)
script_changed = True
continue
config_path = os.path.join(home, "config.toml")
if command_args is None:
@@ -301,7 +512,8 @@ def ensure_codex_home_playwright(codex_home_dir, command_args=None,
"script_path": os.path.join(home, "playwright_mcp.py"),
"config_changed": config_changed,
"script_changed": script_changed,
"script_differs": script_differs}
"script_differs": script_differs,
"script_refreshed": script_refreshed}
def ensure_playwright_mcp_env(codex_home_dir, env):
+186 -3
View File
@@ -147,6 +147,58 @@ _TOOLS = [
},
},
},
{
"name": "browser_snapshot_refs",
"description": (
"Accessibility-style snapshot: interactive elements "
"with stable refs (e.g. e3) plus tag, role and name. "
"Use browser_click_ref to click one; refs are valid "
"until the next snapshot_refs call."
),
"inputSchema": {"type": "object", "properties": {}},
},
{
"name": "browser_click_ref",
"description": (
"Click an element by its ref from browser_snapshot_refs. "
"More robust than a CSS selector across re-renders."
),
"inputSchema": {
"type": "object",
"properties": {"ref": {"type": "string"}},
"required": ["ref"],
},
},
{
"name": "browser_mouse_move",
"description": "Move the mouse to viewport coordinates (1280x800).",
"inputSchema": {
"type": "object",
"properties": {
"x": {"type": "number"},
"y": {"type": "number"},
},
"required": ["x", "y"],
},
},
{
"name": "browser_key_press",
"description": "Press a key (Playwright key name, e.g. Enter, Tab).",
"inputSchema": {
"type": "object",
"properties": {"key": {"type": "string"}},
"required": ["key"],
},
},
{
"name": "browser_type_text",
"description": "Type text at the current focus.",
"inputSchema": {
"type": "object",
"properties": {"text": {"type": "string"}},
"required": ["text"],
},
},
{
"name": "browser_screenshot",
"description": (
@@ -178,6 +230,7 @@ def _err(code, message):
_BROWSER = None
_REMOTE = None
def _get_browser(browser=None):
@@ -189,6 +242,25 @@ def _get_browser(browser=None):
return _BROWSER
def _get_remote(browser=None):
"""Remote control state, lazily created. Keyed by browser
identity so tests passing different fake browsers don't share
one holder flag."""
global _REMOTE
target = _get_browser(browser)
if _REMOTE is not None and _REMOTE.browser is target:
return _REMOTE
try:
from playwright_remote import RemoteState
except ImportError:
try:
from cloud.playwright_remote import RemoteState
except ImportError:
return None
_REMOTE = RemoteState(target)
return _REMOTE
def handle_request(request, browser=None):
"""Handle one JSON-RPC request dict. Returns a response dict or
None for notifications (no id). Tests pass a fake browser."""
@@ -227,6 +299,14 @@ def handle_request(request, browser=None):
name = params.get("name")
args = params.get("arguments") or {}
target = _get_browser(browser)
remote = _get_remote(browser)
if remote is not None and remote.get_holder() == "user":
return {"jsonrpc": "2.0", "id": req_id,
"result": {"content": [{"type": "text",
"text": "error: user holds "
"browser control; wait for "
"release"}],
"isError": True}}
try:
if name == "browser_navigate":
result = target.navigate(args.get("url"))
@@ -239,6 +319,16 @@ def handle_request(request, browser=None):
args.get("text"))
elif name == "browser_click_at":
result = target.click_at(args.get("x"), args.get("y"))
elif name == "browser_mouse_move":
result = target.mouse_move(args.get("x"), args.get("y"))
elif name == "browser_key_press":
result = target.key_press(args.get("key"))
elif name == "browser_type_text":
result = target.type_text(args.get("text"))
elif name == "browser_snapshot_refs":
result = target.snapshot_refs()
elif name == "browser_click_ref":
result = target.click_ref(args.get("ref"))
elif name == "browser_live_shot":
result = target.live_shot(
args.get("output_filename"))
@@ -268,9 +358,75 @@ def handle_request(request, browser=None):
"error": _err(-32601, "method not found: %r" % method)}
def _drain_remote(browser):
"""Main-thread only: run queued user inputs and pump screencast
frames (acks). Called on every loop iteration so remote input
lands even while no JSON-RPC request is in flight."""
remote = _get_remote(browser)
if remote is None:
return
try:
from playwright_remote import execute_input
except ImportError:
try:
from cloud.playwright_remote import execute_input
except ImportError:
return
while True:
try:
item = remote.inputs.get_nowait()
except Exception: # noqa: BLE001 - queue empty
break
if remote.get_holder() != "user":
continue
try:
execute_input(browser, item)
except Exception as exc: # noqa: BLE001 - input errors are data
print("playwright_mcp: remote input error: %s" % exc,
file=sys.stderr)
try:
frame = browser.pump_screencast()
except Exception: # noqa: BLE001 - screencast best-effort
frame = ""
if frame:
remote.set_frame(frame)
def main():
for line in sys.stdin:
line = line.strip()
import select
browser = _get_browser()
remote = _get_remote(browser)
if remote is not None:
try:
from playwright_remote import start_remote_server
except ImportError:
try:
from cloud.playwright_remote import start_remote_server
except ImportError:
start_remote_server = None
if start_remote_server is not None:
socket_path = os.environ.get(
"PLAYWRIGHT_REMOTE_SOCKET",
os.path.join(browser.cwd, ".browser-remote.sock"))
try:
browser.set_frame_sink(_frame_queue())
start_remote_server(remote, socket_path)
except Exception as exc: # noqa: BLE001 - remote best-effort
print("playwright_mcp: remote server failed: %s" % exc,
file=sys.stderr)
buffer = ""
while True:
_drain_remote(browser)
try:
readable, _, _ = select.select([sys.stdin], [], [], 0.05)
except Exception: # noqa: BLE001 - stdin gone
break
if not readable:
continue
chunk = sys.stdin.readline()
if not chunk:
break
line = chunk.strip()
if not line:
continue
try:
@@ -282,7 +438,7 @@ def main():
sys.stdout.flush()
continue
try:
response = handle_request(request)
response = handle_request(request, browser)
except Exception as exc: # noqa: BLE001 - server must not die
print("playwright_mcp: handler error: %s" % exc,
file=sys.stderr)
@@ -296,5 +452,32 @@ def main():
sys.stdout.flush()
def _frame_queue():
try:
from playwright_remote import SimpleQueue
except ImportError:
try:
from cloud.playwright_remote import SimpleQueue
except ImportError:
import collections
import threading
class SimpleQueue: # noqa: D101 - last-resort fallback
def __init__(self):
self._deque = collections.deque()
self._lock = threading.Lock()
def put(self, item):
with self._lock:
self._deque.append(item)
def get_nowait(self):
with self._lock:
if not self._deque:
raise IndexError("queue empty")
return self._deque.popleft()
return SimpleQueue()
if __name__ == "__main__":
main()
+234
View File
@@ -0,0 +1,234 @@
"""Remote-control HTTP service for the Playwright MCP server.
Runs INSIDE the MCP process (same thread-ownership problem as the
browser itself) and listens on a Unix socket in the task worktree
(``.browser-remote.sock`` by default). Why a socket, not a TCP
port: Docker task containers are per-turn and --rm, so a published
port would need a mapping nobody can observe mid-run (the worker's
runner blocks); a socket file in the mounted worktree is visible
to the host control plane with zero mapping, zero conflicts, and
dies with the turn.
Threading contract (measured 2026-10-30: Playwright sync calls
from any thread but the creating one raise "Cannot switch to a
different thread"):
- The HTTP thread NEVER calls page methods. It only reads plain
attributes (Browser.cached_info), the latest frame bytes, and
the holder flag, and it enqueues user inputs.
- The MCP main thread drains the input queue and pumps screencast
frames on every select() iteration (~50ms), so user input lands
within ~50ms even while the agent is idle waiting for its next
tool call, and CDP frame acks keep flowing (no ack = CDP stops
after the first frame).
Control model:
- holder "agent" (default): agent tools work, HTTP input is 409.
- holder "user": the user has taken over (console Takeover
button). Agent browser_* tools fail fast with a plain error so
the agent stops fighting the user; the user releases with
POST /control {"holder": "agent"} and the agent resumes on the
next continue message.
- The socket is chmod 0666: the real gates are the worktree
directory permissions and the control plane's task-owner auth
on its proxy endpoints, not the socket bits.
"""
import collections
import json
import os
import socket
import threading
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
SOCKET_NAME = ".browser-remote.sock"
STREAM_FPS = 5
class SimpleQueue:
"""Minimal thread-safe queue. Deliberately NOT stdlib ``queue``:
cloud/queue.py shadows it whenever cloud/ is on sys.path
(repo mode, tests), breaking ``queue.Queue`` at import time.
Flat-copy tenant runs don't have that file, but this class
works identically in both layouts."""
def __init__(self):
self._deque = collections.deque()
self._lock = threading.Lock()
def put(self, item):
with self._lock:
self._deque.append(item)
def get_nowait(self):
with self._lock:
if not self._deque:
raise IndexError("queue empty")
return self._deque.popleft()
def empty(self):
with self._lock:
return not self._deque
class RemoteState:
"""Shared between the MCP main thread and the HTTP thread."""
def __init__(self, browser):
self.browser = browser
self.holder = "agent"
self._lock = threading.Lock()
self.inputs = SimpleQueue()
self._frame_lock = threading.Lock()
self._frame = ""
def set_holder(self, holder):
if holder not in ("agent", "user"):
raise ValueError("holder must be 'agent' or 'user'")
with self._lock:
self.holder = holder
return self.holder
def get_holder(self):
with self._lock:
return self.holder
def enqueue_input(self, item):
if self.get_holder() != "user":
raise PermissionError(
"user does not hold control (holder=%s); "
"take over first" % self.get_holder())
self.inputs.put(item)
def set_frame(self, data):
with self._frame_lock:
self._frame = data or ""
def get_frame(self):
with self._frame_lock:
return self._frame
def state_dict(self):
info = self.browser.cached_info()
info["holder"] = self.get_holder()
info["streaming"] = bool(self.get_frame())
return info
def execute_input(browser, item):
"""Main-thread only: run one queued user input."""
kind = item.get("type")
if kind == "mouse_click":
return browser.click_at(item.get("x"), item.get("y"))
if kind == "mouse_move":
return browser.mouse_move(item.get("x"), item.get("y"))
if kind == "key":
return browser.key_press(item.get("key"))
if kind == "type":
return browser.type_text(item.get("text", ""))
raise ValueError("unknown input type %r" % (kind,))
def _make_handler(state):
class Handler(BaseHTTPRequestHandler):
protocol_version = "HTTP/1.1"
def log_message(self, *args): # noqa: D102 - quiet
pass
def _send(self, code, obj):
body = json.dumps(obj, ensure_ascii=False).encode("utf-8")
self.send_response(code)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
def _read_json(self):
try:
length = int(self.headers.get("Content-Length") or 0)
except ValueError:
length = 0
raw = self.rfile.read(length) if length > 0 else b""
try:
return json.loads(raw.decode("utf-8")) if raw else {}
except ValueError:
return None
def do_GET(self): # noqa: N802
if self.path == "/state":
self._send(200, state.state_dict())
elif self.path == "/stream":
self._stream()
else:
self._send(404, {"error": "unknown path"})
return
def _stream(self):
import time
self.send_response(200)
self.send_header("Content-Type", "text/event-stream")
self.send_header("Cache-Control", "no-cache")
self.end_headers()
try:
while True:
frame = state.get_frame()
payload = json.dumps(
{"image": frame,
"holder": state.get_holder(),
"ts": int(time.time())},
ensure_ascii=False)
self.wfile.write(
("data: %s\n\n" % payload).encode("utf-8"))
self.wfile.flush()
time.sleep(1.0 / STREAM_FPS)
except (BrokenPipeError, ConnectionResetError):
pass
def do_POST(self): # noqa: N802
body = self._read_json()
if body is None:
self._send(400, {"error": "invalid json"})
return
if self.path == "/control":
try:
holder = state.set_holder(body.get("holder"))
except ValueError as exc:
self._send(400, {"error": str(exc)})
return
self._send(200, {"holder": holder})
elif self.path == "/input":
try:
state.enqueue_input(body)
except PermissionError as exc:
self._send(409, {"error": str(exc)})
return
self._send(200, {"queued": True})
else:
self._send(404, {"error": "unknown path"})
return
return Handler
class UnixHTTPServer(ThreadingHTTPServer):
address_family = socket.AF_UNIX
daemon_threads = True
def server_bind(self):
try:
os.unlink(self.server_address)
except OSError:
pass
super().server_bind()
try:
os.chmod(self.server_address, 0o666)
except OSError:
pass
def start_remote_server(state, socket_path):
"""Start the Unix-socket HTTP service on a daemon thread."""
server = UnixHTTPServer(socket_path, _make_handler(state))
thread = threading.Thread(target=server.serve_forever, daemon=True)
thread.start()
return server
+138
View File
@@ -482,6 +482,55 @@ def _task_worktree_for_files(server, task):
return None, (404, "no local worktree for this task")
def _browser_unix_call(socket_path, method, path, body=None):
"""One HTTP request over a Unix socket (the MCP remote service).
Returns (status, body_bytes). Raw sockets, not http.client:
http.client has no AF_UNIX support, and the SSE stream needs
line-by-line reads on an open connection anyway.
"""
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
sock.settimeout(10)
try:
sock.connect(socket_path)
req = ("%s %s HTTP/1.1\r\nHost: browser\r\n"
"Connection: close\r\n" % (method, path))
if body is not None:
req += ("Content-Type: application/json\r\n"
"Content-Length: %d\r\n" % len(body))
req += "\r\n"
sock.sendall(req.encode("utf-8"))
if body is not None:
sock.sendall(body)
buf = b""
while b"\r\n\r\n" not in buf:
chunk = sock.recv(4096)
if not chunk:
break
buf += chunk
head, _, rest = buf.partition(b"\r\n\r\n")
try:
status = int(head.split(b" ", 2)[1])
except (IndexError, ValueError):
return 502, b'{"error": "bad browser response"}'
length = None
for line in head.split(b"\r\n")[1:]:
if line.lower().startswith(b"content-length:"):
try:
length = int(line.split(b":", 1)[1].strip())
except ValueError:
length = None
data = rest
while length is None or len(data) < length:
chunk = sock.recv(4096)
if not chunk:
break
data += chunk
return status, (data[:length] if length is not None else data)
finally:
sock.close()
def _safe_worktree_path(worktree, rel):
"""worktree 内相对路径的安全解析:空串返回 worktree 本身;
绝对路径、.. 逃逸、符号链接逃逸一律返回 None。realpath 后
@@ -2314,6 +2363,21 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
self._task_timeline(user_id, rest[: -len("/timeline")])
elif rest.endswith("/files") and method == "GET":
self._task_files(user_id, rest[: -len("/files")], query)
elif rest.endswith("/browser/state") and method == "GET":
self._browser_proxy(user_id,
rest[: -len("/browser/state")],
"GET", "/state", None)
elif rest.endswith("/browser/input") and method == "POST":
self._browser_proxy(user_id,
rest[: -len("/browser/input")],
"POST", "/input", self._read_body())
elif rest.endswith("/browser/control") and method == "POST":
self._browser_proxy(user_id,
rest[: -len("/browser/control")],
"POST", "/control", self._read_body())
elif rest.endswith("/browser/stream") and method == "GET":
self._browser_stream(user_id,
rest[: -len("/browser/stream")])
elif rest.endswith("/chat-history") and method == "GET":
self._task_chat_history(
user_id, rest[: -len("/chat-history")])
@@ -5054,6 +5118,80 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
self._send_json(200, {"task_id": task_id, "path": rel,
"size": size, "content": content})
def _browser_socket_for(self, user_id, task_id):
"""Resolve a task's remote-browser Unix socket.
Same auth + worktree resolution as /files. Socket missing
(turn ended, browser never started) is 503, not 404: the
session is transient by design, it will exist again next
turn. Returns (socket_path, None) or (None, (status, msg)).
"""
task, err = self._thread_task(user_id, task_id, "read")
if err is not None:
return None, err
worktree, werr = _task_worktree_for_files(self.server, task)
if werr is not None:
return None, werr
sock_path = os.path.join(worktree, ".browser-remote.sock")
if not os.path.exists(sock_path):
return None, (503, "browser session not active")
return sock_path, None
def _browser_proxy(self, user_id, task_id, method, path, body):
"""Forward one JSON request to the MCP remote service."""
sock_path, err = self._browser_socket_for(user_id, task_id)
if err is not None:
self._send_json(err[0], {"error": err[1]})
return
try:
status, data = _browser_unix_call(sock_path, method, path,
body)
except (OSError, ValueError) as exc:
self._send_json(503, {"error": "browser session unreachable: "
"%s" % exc})
return
try:
obj = json.loads(data.decode("utf-8")) if data else {}
except ValueError:
self._send_json(502, {"error": "bad browser response"})
return
self._send_json(status, obj)
def _browser_stream(self, user_id, task_id):
"""Relay the MCP SSE frame stream to the console client."""
sock_path, err = self._browser_socket_for(user_id, task_id)
if err is not None:
self._send_json(err[0], {"error": err[1]})
return
try:
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
sock.connect(sock_path)
sock.sendall(b"GET /stream HTTP/1.1\r\nHost: browser\r\n\r\n")
buf = b""
while b"\r\n\r\n" not in buf:
chunk = sock.recv(4096)
if not chunk:
raise OSError("browser closed stream")
buf += chunk
except OSError as exc:
self._send_json(503, {"error": "browser session unreachable: "
"%s" % exc})
return
self.send_response(200)
self.send_header("Content-Type", "text/event-stream")
self.send_header("Cache-Control", "no-cache")
self.end_headers()
self._responded = True
try:
stream = sock.makefile("rb")
for line in stream:
self.wfile.write(line)
self.wfile.flush()
except (BrokenPipeError, ConnectionResetError, OSError):
pass
finally:
sock.close()
def _task_chat_history(self, user_id, task_id):
"""GET /tasks/{id}/chat-history:真正的交替历史,一次性
按时间交错拼好返回,前端不再猜顺序。
+13
View File
@@ -34,6 +34,19 @@ screenshot 都针对同一页;要换页再 navigate。
`browser_click_at`,点完必须再 snapshot 一次确认变化,再调
`browser_live_shot` 刷新画面。
## 远控接管(第二期)
用户可以在控制台接管浏览器:接管期间(holder=user)你的
所有 browser_* 工具都会直接报错“user holds browser control”,
不要重试、不要 close 浏览器,停下来等用户交还;交还后
(holder 回到 agent)下一条继续消息会让你续跑,续跑前先
snapshot 一次看清用户改动到哪了。
新工具:`browser_snapshot_refs`(带 ref 的元素清单,点
`browser_click_ref` 比 CSS 选择器稳,页面重渲染后先重新
snapshot_refs 再点);`browser_mouse_move` / `browser_key_press` /
`browser_type_text`(键盘与鼠标,远控输入与 agent 工具同源)。
## 规则
1. 先 navigate,再 snapshot 看实际渲染结果;click / fill 后
+135
View File
@@ -25,6 +25,13 @@ class _FakeLocator:
def inner_text(self, timeout=None):
return self._text
@property
def first(self):
return self
def click(self, timeout=None):
pass
class _FakeMouse:
def __init__(self, page):
@@ -33,6 +40,40 @@ class _FakeMouse:
def click(self, x, y):
self._page.calls.append(("mouse_click", x, y))
def move(self, x, y):
self._page.calls.append(("mouse_move", x, y))
class _FakeKeyboard:
def __init__(self, page):
self._page = page
def press(self, key):
self._page.calls.append(("key_press", key))
def type(self, text):
self._page.calls.append(("key_type", text))
class _FakeCDP:
def __init__(self):
self.sent = []
self.handlers = {}
def on(self, event, handler):
self.handlers[event] = handler
def send(self, method, params=None):
self.sent.append((method, params or {}))
class _FakeContext:
def __init__(self, cdp):
self.cdp = cdp
def new_cdp_session(self, page):
return self.cdp
class _FakePage:
def __init__(self):
@@ -40,6 +81,9 @@ class _FakePage:
self.calls = []
self.shot_path = None
self.mouse = _FakeMouse(self)
self.keyboard = _FakeKeyboard(self)
self.cdp = _FakeCDP()
self.context = _FakeContext(self.cdp)
def title(self):
return "Example Domain"
@@ -60,6 +104,11 @@ class _FakePage:
def evaluate(self, expression):
self.calls.append(("evaluate", expression))
if "querySelectorAll" in expression:
return [{"tag": "button", "role": "", "name": "GO",
"path": "button"},
{"tag": "a", "role": "link", "name": "Home",
"path": "a"}]
return 42
def screenshot(self, path=None):
@@ -166,6 +215,50 @@ class BrowserTest(unittest.TestCase):
with self.assertRaises(BrowserError):
self.browser.click_at("left", 10)
def test_remote_input_methods(self):
self.browser.navigate("https://example.com")
out = self.browser.mouse_move(10, 20)
self.assertTrue(out["ok"])
self.assertIn(("mouse_move", 10, 20), self.page.calls)
out = self.browser.key_press("Enter")
self.assertTrue(out["ok"])
self.assertIn(("key_press", "Enter"), self.page.calls)
out = self.browser.type_text("hi")
self.assertTrue(out["ok"])
self.assertIn(("key_type", "hi"), self.page.calls)
with self.assertRaises(BrowserError):
self.browser.mouse_move(9999, 10)
def test_snapshot_refs_and_click_ref(self):
self.browser.navigate("https://example.com")
out = self.browser.snapshot_refs()
self.assertEqual(out["count"], 2)
self.assertEqual(out["elements"][0]["ref"], "e0")
self.assertEqual(out["elements"][0]["name"], "GO")
clicked = self.browser.click_ref("e0")
self.assertTrue(clicked["ok"])
self.assertEqual(clicked["ref"], "e0")
self.assertIn(("locator", "button"), self.page.calls)
with self.assertRaises(BrowserError):
self.browser.click_ref("e99")
def test_screencast_attach_and_pump(self):
from playwright_remote import SimpleQueue
sink = SimpleQueue()
self.browser.set_frame_sink(sink)
self.browser.navigate("https://example.com")
self.assertIn(("Page.startScreencast",
{"format": "jpeg", "quality": 60,
"maxWidth": 1280, "maxHeight": 800,
"everyNthFrame": 2}),
[(m, p) for m, p in self.page.cdp.sent])
handler = self.page.cdp.handlers["Page.screencastFrame"]
handler({"sessionId": "s1", "data": "FRAMEDATA"})
frame = self.browser.pump_screencast()
self.assertEqual(frame, "FRAMEDATA")
self.assertIn(("Page.screencastFrameAck", {"sessionId": "s1"}),
[(m, p) for m, p in self.page.cdp.sent])
def test_live_shot_writes_fixed_file_and_status(self):
self.browser.navigate("https://example.com")
out = self.browser.live_shot()
@@ -192,6 +285,22 @@ class FakeMCPBrowser:
"path": "/w/browser-live.png", "bytes": 456,
"url": "https://x/", "title": "T", "timestamp": 1}
def mouse_move(self, x, y):
return {"ok": True, "x": x, "y": y}
def key_press(self, key):
return {"ok": True, "key": key}
def type_text(self, text):
return {"ok": True}
def snapshot_refs(self):
return {"elements": [{"ref": "e0", "tag": "button",
"role": "", "name": "GO"}], "count": 1}
def click_ref(self, ref):
return {"ok": True, "ref": ref}
def snapshot(self):
return {"url": "https://x/", "title": "T", "text": "hi",
"truncated": False}
@@ -223,6 +332,32 @@ class MCPTest(unittest.TestCase):
self.assertIn("https://example.com", text)
self.assertFalse(resp["result"]["isError"])
def test_holder_user_blocks_agent_tools(self):
import playwright_mcp
fake = FakeMCPBrowser()
remote = playwright_mcp._get_remote(fake)
remote.set_holder("user")
try:
resp = handle_request(
{"jsonrpc": "2.0", "id": 3, "method": "tools/call",
"params": {"name": "browser_navigate",
"arguments": {"url": "https://example.com"}}},
browser=fake)
self.assertTrue(resp["result"]["isError"])
self.assertIn("holds browser control",
resp["result"]["content"][0]["text"])
finally:
remote.set_holder("agent")
def test_new_tools_listed(self):
listed = handle_request({"jsonrpc": "2.0", "id": 4,
"method": "tools/list"})
names = [t["name"] for t in listed["result"]["tools"]]
for name in ("browser_snapshot_refs", "browser_click_ref",
"browser_mouse_move", "browser_key_press",
"browser_type_text"):
self.assertIn(name, names)
if __name__ == "__main__":
unittest.main()
+156
View File
@@ -0,0 +1,156 @@
"""Tests for the Playwright remote-control service (fake only).
Covers: holder transitions + input gating, the Unix-socket HTTP
endpoints (state/control/input/stream), and the main-thread
input execution path. No real Chromium: the CDP/screencast path
is exercised through fakes, real-browser proof lives in the
production verification task.
"""
import json
import os
import socket
import sys
import tempfile
import threading
import unittest
import urllib.request
sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
from playwright_remote import (RemoteState, execute_input, # noqa: E402
start_remote_server)
class _StubBrowser:
def __init__(self):
self.calls = []
def cached_info(self):
return {"url": "https://example.com", "title": "T",
"started": True}
def click_at(self, x, y):
self.calls.append(("click_at", x, y))
return {"ok": True}
def mouse_move(self, x, y):
self.calls.append(("mouse_move", x, y))
return {"ok": True}
def key_press(self, key):
self.calls.append(("key_press", key))
return {"ok": True}
def type_text(self, text):
self.calls.append(("type_text", text))
return {"ok": True}
class RemoteStateTest(unittest.TestCase):
def test_holder_and_input_gating(self):
state = RemoteState(_StubBrowser())
self.assertEqual(state.get_holder(), "agent")
with self.assertRaises(PermissionError):
state.enqueue_input({"type": "mouse_click", "x": 1, "y": 2})
state.set_holder("user")
state.enqueue_input({"type": "mouse_click", "x": 10, "y": 20})
item = state.inputs.get_nowait()
execute_input(state.browser, item)
self.assertIn(("click_at", 10, 20), state.browser.calls)
with self.assertRaises(ValueError):
state.set_holder("nobody")
with self.assertRaises(ValueError):
execute_input(state.browser, {"type": "teleport"})
def test_state_dict(self):
state = RemoteState(_StubBrowser())
state.set_frame("abc")
d = state.state_dict()
self.assertEqual(d["holder"], "agent")
self.assertTrue(d["streaming"])
self.assertEqual(d["url"], "https://example.com")
class HttpServiceTest(unittest.TestCase):
def setUp(self):
self._tmp = tempfile.TemporaryDirectory()
self.browser = _StubBrowser()
self.state = RemoteState(self.browser)
self.sock = os.path.join(self._tmp.name, "remote.sock")
self.server = start_remote_server(self.state, self.sock)
def tearDown(self):
self.server.shutdown()
self.server.server_close()
self._tmp.cleanup()
def _unix_get(self, path):
import http.client
class UnixConn(http.client.HTTPConnection):
def connect(self):
self.sock = socket.socket(socket.AF_UNIX,
socket.SOCK_STREAM)
self.sock.connect(self._sock_path)
conn = UnixConn("localhost", timeout=5)
conn._sock_path = self.sock
conn.request("GET", path)
resp = conn.getresponse()
return resp.status, json.loads(resp.read().decode("utf-8"))
def _unix_post(self, path, obj):
import http.client
class UnixConn(http.client.HTTPConnection):
def connect(self):
self.sock = socket.socket(socket.AF_UNIX,
socket.SOCK_STREAM)
self.sock.connect(self._sock_path)
conn = UnixConn("localhost", timeout=5)
conn._sock_path = self.sock
body = json.dumps(obj).encode("utf-8")
conn.request("POST", path, body=body,
headers={"Content-Type": "application/json"})
resp = conn.getresponse()
return resp.status, json.loads(resp.read().decode("utf-8"))
def test_state_control_input_flow(self):
status, body = self._unix_get("/state")
self.assertEqual(status, 200)
self.assertEqual(body["holder"], "agent")
status, body = self._unix_post(
"/input", {"type": "mouse_click", "x": 1, "y": 2})
self.assertEqual(status, 409)
status, body = self._unix_post("/control", {"holder": "user"})
self.assertEqual((status, body["holder"]), (200, "user"))
status, body = self._unix_post(
"/input", {"type": "key", "key": "Enter"})
self.assertEqual((status, body["queued"]), (200, True))
self.assertEqual(self.state.inputs.get_nowait()["key"], "Enter")
status, body = self._unix_post("/control", {"holder": "agent"})
self.assertEqual(body["holder"], "agent")
def test_stream_emits_latest_frame(self):
self.state.set_frame("FRAMEBYTES")
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
sock.settimeout(5)
sock.connect(self.sock)
sock.sendall(b"GET /stream HTTP/1.1\r\nHost: x\r\n\r\n")
buf = b""
while b"\n\n" not in buf:
buf += sock.recv(4096)
head, _, rest = buf.partition(b"\n\n")
self.assertIn(b"text/event-stream", head)
first = rest
while b"\n\n" not in first:
first += sock.recv(4096)
payload = json.loads(first.split(b"data: ", 1)[1].split(b"\n\n")[0])
self.assertEqual(payload["image"], "FRAMEBYTES")
sock.close()
if __name__ == "__main__":
unittest.main()
+140
View File
@@ -0,0 +1,140 @@
"""Browser proxy endpoints: /tasks/{id}/browser/{state,input,control}.
The control plane forwards to the MCP remote service over the Unix
socket in the task worktree. Here the socket server is the real
playwright_remote module with a stub browser; no Chromium, no
worker, no model.
"""
import http.client
import json
import os
import sqlite3
import sys
import tempfile
import threading
import unittest
from pathlib import Path
try:
from cloud.auth import AuthStore
from cloud.playwright_remote import RemoteState, start_remote_server
from cloud.server import create_server
except ImportError:
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
from cloud.auth import AuthStore
from cloud.playwright_remote import RemoteState, start_remote_server
from cloud.server import create_server
class _StubBrowser:
def cached_info(self):
return {"url": "https://example.com", "title": "T",
"started": True}
class BrowserProxyTestCase(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.db_path = os.path.join(self.tmp.name, "control.db")
self.auth = AuthStore(self.db_path)
self.owner_id, self.owner_key = self.auth.create_user("owner")
self.outsider_id, self.outsider_key = self.auth.create_user("o")
self.server = create_server(self.db_path, "127.0.0.1", 0)
self.worktrees = os.path.join(self.tmp.name, "host-worktrees")
os.makedirs(self.worktrees)
self.server.host_worktrees_dir = self.worktrees
self.port = self.server.server_address[1]
self.thread = threading.Thread(
target=self.server.serve_forever, daemon=True)
self.thread.start()
status, self.project = self._req(
"POST", "/projects", {"name": "demo"}, self.owner_key)
self.assertEqual(status, 201)
self._task("task-b1")
wt = os.path.join(self.worktrees, "task-b1")
os.makedirs(wt)
self.remote = RemoteState(_StubBrowser())
self.sock_server = start_remote_server(
self.remote, os.path.join(wt, ".browser-remote.sock"))
self._task("task-nosock")
os.makedirs(os.path.join(self.worktrees, "task-nosock"))
def tearDown(self):
self.sock_server.shutdown()
self.sock_server.server_close()
self.server.shutdown()
self.server.server_close()
self.thread.join(timeout=5)
try:
self.auth.conn.close()
except Exception:
pass
self.tmp.cleanup()
def _task(self, task_id):
conn = sqlite3.connect(self.db_path)
try:
conn.execute(
"INSERT INTO tasks(id, project_id, prompt, status)"
" VALUES(?, ?, ?, ?)",
(task_id, self.project["id"], "do it", "running"))
conn.commit()
finally:
conn.close()
def _req(self, method, path, body=None, key=None):
conn = http.client.HTTPConnection("127.0.0.1", self.port, timeout=10)
hdrs = {}
payload = None
if body is not None:
payload = json.dumps(body).encode("utf-8")
hdrs["Content-Type"] = "application/json"
if key is not None:
hdrs["X-API-Key"] = key
try:
conn.request(method, path, body=payload, headers=hdrs)
resp = conn.getresponse()
data = resp.read()
finally:
conn.close()
try:
parsed = json.loads(data.decode("utf-8")) if data else None
except ValueError:
parsed = None
return resp.status, parsed
def test_state_input_control_flow(self):
status, body = self._req(
"GET", "/tasks/task-b1/browser/state", key=self.owner_key)
self.assertEqual(status, 200, body)
self.assertEqual(body["holder"], "agent")
self.assertEqual(body["url"], "https://example.com")
status, body = self._req(
"POST", "/tasks/task-b1/browser/input",
{"type": "mouse_click", "x": 1, "y": 2}, self.owner_key)
self.assertEqual(status, 409, body)
status, body = self._req(
"POST", "/tasks/task-b1/browser/control",
{"holder": "user"}, self.owner_key)
self.assertEqual((status, body["holder"]), (200, "user"))
status, body = self._req(
"POST", "/tasks/task-b1/browser/input",
{"type": "key", "key": "Enter"}, self.owner_key)
self.assertEqual((status, body["queued"]), (200, True))
status, body = self._req(
"POST", "/tasks/task-b1/browser/control",
{"holder": "agent"}, self.owner_key)
self.assertEqual(body["holder"], "agent")
def test_forbidden_and_no_session(self):
status, _ = self._req(
"GET", "/tasks/task-b1/browser/state", key=self.outsider_key)
self.assertEqual(status, 403)
status, body = self._req(
"GET", "/tasks/task-nosock/browser/state", key=self.owner_key)
self.assertEqual(status, 503, body)
if __name__ == "__main__":
unittest.main()
+9 -2
View File
@@ -2631,13 +2631,20 @@ class Worker:
# 与 imagegen 同款:播种失败直接抛错、任务不派发
# ——静默无浏览器比 loud 失败更糟,尤其对已宣传
# “能开浏览器”的任务。代理与浏览器路径由
# _prepare_tenant 的 env 块给。
_ensure_playwright_mcp(
# _prepare_tenant 的 env 块给。MCP 脚本是系统文件:
# 播种器在版本变化时刷新老租户的旧脚本(见
# ensure_codex_home_playwright),刷新记录打日志。
_seeded = _ensure_playwright_mcp(
dirs["codex_home"],
command_args=[
"python3",
"/var/lib/codex-app-server/codex-home/"
"playwright_mcp.py"])
if _seeded.get("script_refreshed"):
self.logger.info(
"playwright MCP scripts refreshed for project %s: %s",
task.get("project_id"),
",".join(_seeded["script_refreshed"]))
if self.seed_skills:
# 技能播种失败绝不阻塞任务:技能是可选的流程
# 说明,不是 websearch 那种“已宣传就必须有”的