Merge pull request #180 from openqj/feat-browser-phase1

feat: 浏览器一期：坐标点击 + live 截图

cloud/playwright_browser.py

@@ -148,6 +148,59 @@ class Browser:
         result["ok"] = True
         return result
 
+    def click_at(self, x, y):
+        """Click by viewport coordinates (CSS pixels, origin at the
+        top-left of the 1280x800 viewport). This is the tool the
+        console live view drives: the user clicks on a screenshot
+        and the coordinates arrive here. Bounds-checked so a
+        mis-scaled click cannot scroll the page into nowhere."""
+        try:
+            fx = float(x)
+            fy = float(y)
+        except (TypeError, ValueError):
+            raise BrowserError("click_at needs numeric x/y, got %r/%r"
+                               % (x, y))
+        if not (0 <= fx <= 1280 and 0 <= fy <= 800):
+            raise BrowserError(
+                "click_at out of viewport 1280x800: %s,%s" % (x, y))
+        self._ensure()
+        try:
+            self._page.mouse.click(fx, fy)
+            self._current_url = self._page.url
+        except Exception as exc:  # noqa: BLE001
+            raise BrowserError("click_at failed: %s" % exc)
+        result = self._info()
+        result["ok"] = True
+        result["x"] = fx
+        result["y"] = fy
+        return result
+
+    def live_shot(self, output_filename=None):
+        """Screenshot to a FIXED live-view file plus a JSON status
+        sidecar, so a watcher (console, user) can poll one stable
+        path instead of chasing per-shot filenames. The agent
+        should call this after every navigate/click/fill while a
+        live view is wanted. Returns path, bytes, url, title and
+        a unix timestamp."""
+        self._ensure()
+        name = _validate_basename(
+            output_filename or "browser-live.png", "output_filename")
+        path = os.path.join(self.cwd, name)
+        try:
+            self._page.screenshot(path=path)
+        except Exception as exc:  # noqa: BLE001
+            raise BrowserError("live_shot failed: %s" % exc)
+        info = self._info()
+        status = {"image": name, "path": path,
+                  "bytes": os.path.getsize(path),
+                  "url": info["url"], "title": info["title"],
+                  "timestamp": int(time.time())}
+        status_path = os.path.join(self.cwd, "browser-live.json")
+        with open(status_path, "w", encoding="utf-8") as fh:
+            json.dump(status, fh, ensure_ascii=False)
+        status["status_path"] = status_path
+        return status
+
     def evaluate(self, expression):
         if not expression or not str(expression).strip():
             raise BrowserError("expression is required")

cloud/playwright_mcp.py

@@ -109,6 +109,44 @@ _TOOLS = [
             "required": ["expression"],
         },
     },
+    {
+        "name": "browser_click_at",
+        "description": (
+            "Click at viewport coordinates (CSS pixels, origin "
+            "top-left of the 1280x800 viewport). This is the tool "
+            "a live viewer drives: the user clicks on a screenshot "
+            "and the coordinates arrive here. Then snapshot to see "
+            "the result."
+        ),
+        "inputSchema": {
+            "type": "object",
+            "properties": {
+                "x": {"type": "number"},
+                "y": {"type": "number"},
+            },
+            "required": ["x", "y"],
+        },
+    },
+    {
+        "name": "browser_live_shot",
+        "description": (
+            "Screenshot to the FIXED live-view file "
+            "browser-live.png plus a browser-live.json status "
+            "sidecar (url, title, timestamp), so a watcher can "
+            "poll one stable path. Call after every navigate / "
+            "click / fill while a live view is wanted. Returns "
+            "path, bytes, url, title and timestamp."
+        ),
+        "inputSchema": {
+            "type": "object",
+            "properties": {
+                "output_filename": {"type": "string",
+                                    "description": "Plain basename "
+                                    "to save as. Default: "
+                                    "browser-live.png"},
+            },
+        },
+    },
     {
         "name": "browser_screenshot",
         "description": (
@@ -199,6 +237,11 @@ def handle_request(request, browser=None):
             elif name == "browser_fill":
                 result = target.fill(args.get("selector"),
                                      args.get("text"))
+            elif name == "browser_click_at":
+                result = target.click_at(args.get("x"), args.get("y"))
+            elif name == "browser_live_shot":
+                result = target.live_shot(
+                    args.get("output_filename"))
             elif name == "browser_evaluate":
                 result = target.evaluate(args.get("expression"))
             elif name == "browser_screenshot":

cloud/skills/browser-automation/SKILL.md

@@ -17,11 +17,23 @@ description: >
 `browser_fill`（按选择器填表）、
 `browser_evaluate`（在页内跑 JS 表达式，返回字符串，限 4000 字）、
 `browser_screenshot`（把当前页存成工作区内的 PNG，返回绝对路径与字节数）、
+`browser_click_at`（按视口坐标点击，坐标系为 1280x800、原点左上角，
+是 live 视图里用户点击的落点）、
+`browser_live_shot`（把当前画面存成固定文件 browser-live.png，
+附带 browser-live.json 状态：URL、标题、时间戳，供 watcher 轮询）、
 `browser_close`（关闭浏览器，可重复调用）。
 
 浏览器是单页的：一次 navigate，之后 snapshot / click / fill /
 screenshot 都针对同一页；要换页再 navigate。
 
+## live 视图（第一期）
+
+有人在看浏览器时，每次 navigate / click / fill / click_at 之后
+都要调一次 `browser_live_shot`，让 browser-live.png 保持最新；
+不要只在最后截图一次。用户在 live 图上点的坐标会以 x/y 到达
+`browser_click_at`，点完必须再 snapshot 一次确认变化，再调
+`browser_live_shot` 刷新画面。
+
 ## 规则
 
 1. 先 navigate，再 snapshot 看实际渲染结果；click / fill 后

cloud/tests/test_playwright.py

@@ -26,11 +26,20 @@ class _FakeLocator:
         return self._text
 
 
+class _FakeMouse:
+    def __init__(self, page):
+        self._page = page
+
+    def click(self, x, y):
+        self._page.calls.append(("mouse_click", x, y))
+
+
 class _FakePage:
     def __init__(self):
         self.url = "https://example.com/"
         self.calls = []
         self.shot_path = None
+        self.mouse = _FakeMouse(self)
 
     def title(self):
         return "Example Domain"
@@ -146,11 +155,43 @@ class BrowserTest(unittest.TestCase):
         self.assertTrue(self.browser.close()["closed"])
         self.assertTrue(self.browser.close()["closed"])
 
+    def test_click_at_and_bounds(self):
+        self.browser.navigate("https://example.com")
+        out = self.browser.click_at(100, 200)
+        self.assertTrue(out["ok"])
+        self.assertEqual(out["x"], 100)
+        self.assertIn(("mouse_click", 100, 200), self.page.calls)
+        with self.assertRaises(BrowserError):
+            self.browser.click_at(2000, 10)
+        with self.assertRaises(BrowserError):
+            self.browser.click_at("left", 10)
+
+    def test_live_shot_writes_fixed_file_and_status(self):
+        self.browser.navigate("https://example.com")
+        out = self.browser.live_shot()
+        self.assertEqual(out["image"], "browser-live.png")
+        self.assertTrue(out["bytes"] > 0)
+        self.assertTrue(os.path.exists(out["path"]))
+        with open(os.path.join(self._tmp.name,
+                               "browser-live.json")) as fh:
+            status = json.load(fh)
+        self.assertEqual(status["image"], "browser-live.png")
+        self.assertEqual(status["url"], "https://example.com")
+
 
 class FakeMCPBrowser:
     def navigate(self, url):
         return {"url": url, "title": "T", "ok": True}
 
+    def click_at(self, x, y):
+        return {"url": "https://x/", "title": "T", "ok": True,
+                "x": x, "y": y}
+
+    def live_shot(self, name=None):
+        return {"image": name or "browser-live.png",
+                "path": "/w/browser-live.png", "bytes": 456,
+                "url": "https://x/", "title": "T", "timestamp": 1}
+
     def snapshot(self):
         return {"url": "https://x/", "title": "T", "text": "hi",
                 "truncated": False}
@@ -168,7 +209,8 @@ class MCPTest(unittest.TestCase):
         for name in ("browser_navigate", "browser_snapshot",
                      "browser_click", "browser_fill",
                      "browser_evaluate", "browser_screenshot",
-                     "browser_close"):
+                     "browser_close", "browser_click_at",
+                     "browser_live_shot"):
             self.assertIn(name, names)
 
     def test_tools_call_dispatches(self):