feat: 浏览器二期后端：远控服务 + ref 快照 + server 代理

- playwright_remote.py（新）：MCP 进程内 Unix socket HTTP 服务，监听 worktree/.browser-remote.sock；GET /state、GET /stream （SSE 画面流）、POST /input、POST /control（接管/交还）。线程契约（实测）：Playwright 同步调用只能在创建页面的线程，所以 HTTP 线程只读属性/读最新帧/入队，MCP 主循环用 select 轮询（50ms） draining 输入并 pump screencast 帧（无 ack 则 CDP 首帧后停流）。 - 画面流走 CDP screencast 帧（学 chrome-devtools-mcp），不用定时截图；ffmpeg 录像暂不做。 - snapshot_refs/click_ref（学 playwright-mcp 的 ref 机制）+ mouse_move/key_press/type_text；agent 工具在 holder=user 时直接报错，等用户交还。 - server.py：/tasks/{id}/browser/{state,stream,input,control} 代理到 worktree socket，复用任务鉴权与 worktree 解析；会话不存在时 503（轮次结束即消失是设计）。 - 播种器：MCP 脚本是系统文件，版本变化时刷新老租户旧脚本（script_refreshed 打日志），否则远控只对新项目生效；新增 playwright_remote.py 进播种清单。 - 队列遮蔽修复：cloud/queue.py 会遮蔽 stdlib queue，改用 SimpleQueue（deque+锁），仓库模式与 flat-copy 两路都通。 - 测试：新 20 项（remote 服务、server 代理、ref/输入/ screencast fake），全量 1354 项通过（skipped 4）。
2026-10-30 06:44:24 +08:00
1 parent 7bea8df512
commit 6f39ab3945
@@ -61,6 +61,16 @@ class Browser:
        self._browser = None
        self._page = None
        self._current_url = ""
+        self._last_title = ""
+        # ref snapshot state (playwright-mcp style): ref -> css path
+        self._ref_paths = {}
+        # screencast sink: a queue.Queue set by the remote service.
+        # The CDP frame callback only enqueues; the MCP main thread
+        # drains it (pump_screencast) because Playwright sync calls
+        # must run on the thread that created the page.
+        self._frame_sink = None
+        self._cdp = None
+        self._cdp_page = None

    def _ensure(self):
        if self._page is not None:
@@ -92,8 +102,15 @@ class Browser:
            title = self._page.title()
        except Exception:  # noqa: BLE001 - title is best-effort
            title = ""
+        self._last_title = title
        return {"url": self._current_url, "title": title}

+    def cached_info(self):
+        """Plain-attribute state, safe to read from any thread
+        (the remote HTTP thread must never call page.title())."""
+        return {"url": self._current_url, "title": self._last_title,
+                "started": self._page is not None}
+
    def navigate(self, url):
        if not url or not str(url).strip():
            raise BrowserError("url is required")
@@ -102,6 +119,7 @@ class Browser:
            self._page.goto(str(url), timeout=NAV_TIMEOUT_MS,
                            wait_until="domcontentloaded")
            self._current_url = self._page.url
+            self._start_screencast()
        except Exception as exc:  # noqa: BLE001
            raise BrowserError("navigate failed: %s" % exc)
        result = self._info()
@@ -148,6 +166,185 @@ class Browser:
        result["ok"] = True
        return result

+    def mouse_move(self, x, y):
+        try:
+            fx = float(x)
+            fy = float(y)
+        except (TypeError, ValueError):
+            raise BrowserError("mouse_move needs numeric x/y, got %r/%r"
+                               % (x, y))
+        if not (0 <= fx <= 1280 and 0 <= fy <= 800):
+            raise BrowserError(
+                "mouse_move out of viewport 1280x800: %s,%s" % (x, y))
+        self._ensure()
+        try:
+            self._page.mouse.move(fx, fy)
+        except Exception as exc:  # noqa: BLE001
+            raise BrowserError("mouse_move failed: %s" % exc)
+        result = self._info()
+        result["ok"] = True
+        result["x"] = fx
+        result["y"] = fy
+        return result
+
+    def key_press(self, key):
+        if not key or not str(key).strip():
+            raise BrowserError("key is required")
+        self._ensure()
+        try:
+            self._page.keyboard.press(str(key))
+        except Exception as exc:  # noqa: BLE001
+            raise BrowserError("key_press failed: %s" % exc)
+        result = self._info()
+        result["ok"] = True
+        result["key"] = str(key)
+        return result
+
+    def type_text(self, text):
+        if text is None:
+            raise BrowserError("text is required")
+        self._ensure()
+        try:
+            self._page.keyboard.type(str(text))
+        except Exception as exc:  # noqa: BLE001
+            raise BrowserError("type_text failed: %s" % exc)
+        result = self._info()
+        result["ok"] = True
+        return result
+
+    def snapshot_refs(self):
+        """Accessibility-style snapshot with refs (playwright-mcp
+        style): interactive elements get a stable ref valid until
+        the next snapshot_refs call; click_ref(ref) clicks one.
+        The ref maps to a computed CSS path, so it survives
+        re-renders better than an index into a stale list."""
+        self._ensure()
+        script = """
+() => {
+  const path = (el) => {
+    const parts = [];
+    while (el && el.nodeType === 1 && el !== document.body) {
+      let tag = el.tagName.toLowerCase();
+      const parent = el.parentElement;
+      if (parent) {
+        const sibs = Array.from(parent.children).filter(
+          c => c.tagName === el.tagName);
+        if (sibs.length > 1)
+          tag += ':nth-of-type(' + (sibs.indexOf(el) + 1) + ')';
+      }
+      parts.unshift(tag);
+      el = parent;
+    }
+    return parts.join(' > ');
+  };
+  const out = [];
+  const els = document.querySelectorAll(
+    'a, button, input, select, textarea, [role="button"], ' +
+    '[role="link"], [role="textbox"], [role="checkbox"], ' +
+    '[role="radio"], [role="combobox"]');
+  els.forEach((el) => {
+    const r = el.getBoundingClientRect();
+    if (r.width === 0 && r.height === 0) return;
+    const name = (el.getAttribute('aria-label') ||
+                  el.textContent || el.value || el.placeholder || '')
+      .trim().replace(/\\s+/g, ' ').slice(0, 80);
+    out.push({tag: el.tagName.toLowerCase(),
+              role: el.getAttribute('role') || '',
+              name: name, path: path(el)});
+  });
+  return out;
+}
+"""
+        try:
+            elements = self._page.evaluate(script)
+        except Exception as exc:  # noqa: BLE001
+            raise BrowserError("snapshot_refs failed: %s" % exc)
+        self._ref_paths = {}
+        items = []
+        for i, el in enumerate(elements or []):
+            ref = "e%d" % i
+            self._ref_paths[ref] = el.get("path", "")
+            items.append({"ref": ref, "tag": el.get("tag", ""),
+                          "role": el.get("role", ""),
+                          "name": el.get("name", "")})
+        result = self._info()
+        result["elements"] = items
+        result["count"] = len(items)
+        return result
+
+    def click_ref(self, ref):
+        if not ref or ref not in self._ref_paths:
+            raise BrowserError(
+                "unknown ref %r: run browser_snapshot_refs first" % (ref,))
+        self._ensure()
+        path = self._ref_paths[ref]
+        try:
+            self._page.locator(path).first.click(timeout=10000)
+            self._current_url = self._page.url
+        except Exception as exc:  # noqa: BLE001
+            raise BrowserError("click_ref failed: %s" % exc)
+        result = self._info()
+        result["ok"] = True
+        result["ref"] = ref
+        return result
+
+    def set_frame_sink(self, sink):
+        """Remote service: frames arrive here via CDP callback."""
+        self._frame_sink = sink
+        if self._page is not None:
+            self._start_screencast()
+
+    def _start_screencast(self):
+        if self._frame_sink is None or self._page is None:
+            return
+        if self._cdp is not None and self._cdp_page is self._page:
+            return
+        try:
+            cdp = self._page.context.new_cdp_session(self._page)
+
+            def _on_frame(params):
+                try:
+                    self._frame_sink.put(
+                        (params.get("sessionId", ""),
+                         params.get("data", "")))
+                except Exception:  # noqa: BLE001 - callback must not die
+                    pass
+
+            cdp.on("Page.screencastFrame", _on_frame)
+            cdp.send("Page.startScreencast",
+                     {"format": "jpeg", "quality": 60,
+                      "maxWidth": 1280, "maxHeight": 800,
+                      "everyNthFrame": 2})
+            self._cdp = cdp
+            self._cdp_page = self._page
+        except Exception:  # noqa: BLE001 - screencast is best-effort
+            self._cdp = None
+            self._cdp_page = None
+
+    def pump_screencast(self):
+        """Main-thread only: store the latest frame and ack every
+        queued frame (no ack = CDP stops sending after the first).
+        Returns the latest frame data (base64 jpeg) or ''."""
+        if self._frame_sink is None:
+            return ""
+        latest = ""
+        while True:
+            try:
+                sid, data = self._frame_sink.get_nowait()
+            except Exception:  # noqa: BLE001 - queue empty
+                break
+            if data:
+                latest = data
+            if self._cdp is not None:
+                try:
+                    self._cdp.send("Page.screencastFrameAck",
+                                   {"sessionId": sid})
+                except Exception:  # noqa: BLE001 - ack best-effort
+                    pass
+        if latest:
+            self._latest_frame = latest
+        return getattr(self, "_latest_frame", "")
+
    def click_at(self, x, y):
        """Click by viewport coordinates (CSS pixels, origin at the
        top-left of the 1280x800 viewport). This is the tool the
@@ -226,6 +423,13 @@ class Browser:

    def close(self):
        errors = []
+        if self._cdp is not None:
+            try:
+                self._cdp.send("Page.stopScreencast")
+            except Exception:  # noqa: BLE001 - best-effort
+                pass
+            self._cdp = None
+            self._cdp_page = None
        for closer in (lambda: self._page and self._page.close(),
                       lambda: self._browser and self._browser.close(),
                       lambda: self._pw and self._pw.stop()):
@@ -245,11 +449,14 @@ def ensure_codex_home_playwright(codex_home_dir, command_args=None,
                                 script_dir=None):
    """Seed playwright MCP files + config into a tenant codex-home.

-    Mirrors ``ensure_codex_home_imagegen``: copies
-    ``playwright_browser.py`` and ``playwright_mcp.py`` only when
-    missing (a tenant-modified script is never silently
-    overwritten, it is reported in ``script_differs``), and
-    appends ``[mcp_servers.playwright]`` only when absent. No
+    MCP scripts are SYSTEM files, not tenant data: a tenant never
+    edits them (user files live in the worktree, not codex-home),
+    so unlike the old never-overwrite rule, a differing script is
+    REFRESHED to the deployed version and reported in
+    ``script_refreshed``. Without this, every MCP feature ships
+    dead for existing projects: they keep the first-seeded script
+    forever and only brand-new projects get the update.
+    Appends ``[mcp_servers.playwright]`` only when absent. No
    secrets live here -- proxy vars arrive via
    ``ensure_playwright_mcp_env`` from worker proxy_env, and the
    browsers path is a fixed image path.
@@ -261,7 +468,9 @@ def ensure_codex_home_playwright(codex_home_dir, command_args=None,
    here = os.path.abspath(script_dir or os.path.dirname(__file__))
    script_changed = False
    script_differs = []
-    for name in ("playwright_browser.py", "playwright_mcp.py"):
+    script_refreshed = []
+    for name in ("playwright_browser.py", "playwright_mcp.py",
+                 "playwright_remote.py"):
        src = os.path.join(here, name)
        dst = os.path.join(home, name)
        if not os.path.exists(src):
@@ -275,7 +484,9 @@ def ensure_codex_home_playwright(codex_home_dir, command_args=None,
            with open(dst, "rb") as fh:
                dst_bytes = fh.read()
            if src_bytes != dst_bytes:
-                script_differs.append(name)
+                shutil.copyfile(src, dst)
+                script_refreshed.append(name)
+                script_changed = True
                continue
    config_path = os.path.join(home, "config.toml")
    if command_args is None:
@@ -301,7 +512,8 @@ def ensure_codex_home_playwright(codex_home_dir, command_args=None,
            "script_path": os.path.join(home, "playwright_mcp.py"),
            "config_changed": config_changed,
            "script_changed": script_changed,
-            "script_differs": script_differs}
+            "script_differs": script_differs,
+            "script_refreshed": script_refreshed}


 def ensure_playwright_mcp_env(codex_home_dir, env):