From 814fe8f7c28404cd6fa3065dd130270b33e88651 Mon Sep 17 00:00:00 2001 From: host Date: Fri, 30 Oct 2026 06:19:55 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=B5=8F=E8=A7=88=E5=99=A8=E4=B8=80?= =?UTF-8?q?=E6=9C=9F=EF=BC=9A=E5=9D=90=E6=A0=87=E7=82=B9=E5=87=BB=20+=20li?= =?UTF-8?q?ve=20=E6=88=AA=E5=9B=BE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - browser_click_at(x, y)：视口坐标点击（1280x800、左上角 原点），越界与非数字拒绝；live 视图里用户在截图上的 点击落点就走这个工具。 - browser_live_shot：固定文件 browser-live.png + 状态 sidecar browser-live.json（URL/标题/时间戳），watcher 轮询一个稳定路径即可；技能要求每次动作后都调一次， 保持画面最新。 - MCP 新增 browser_click_at / browser_live_shot 两个工具； 技能补 live 视图一节；测试 9 项（含 fake mouse、越界、 live 文件与状态读回）。 --- cloud/playwright_browser.py | 53 ++++++++++++++++++++++++ cloud/playwright_mcp.py | 43 +++++++++++++++++++ cloud/skills/browser-automation/SKILL.md | 12 ++++++ cloud/tests/test_playwright.py | 44 +++++++++++++++++++- 4 files changed, 151 insertions(+), 1 deletion(-) diff --git a/cloud/playwright_browser.py b/cloud/playwright_browser.py index ddb5e69..f5eaba0 100644 --- a/cloud/playwright_browser.py +++ b/cloud/playwright_browser.py @@ -148,6 +148,59 @@ class Browser: result["ok"] = True return result + def click_at(self, x, y): + """Click by viewport coordinates (CSS pixels, origin at the + top-left of the 1280x800 viewport). This is the tool the + console live view drives: the user clicks on a screenshot + and the coordinates arrive here. Bounds-checked so a + mis-scaled click cannot scroll the page into nowhere.""" + try: + fx = float(x) + fy = float(y) + except (TypeError, ValueError): + raise BrowserError("click_at needs numeric x/y, got %r/%r" + % (x, y)) + if not (0 <= fx <= 1280 and 0 <= fy <= 800): + raise BrowserError( + "click_at out of viewport 1280x800: %s,%s" % (x, y)) + self._ensure() + try: + self._page.mouse.click(fx, fy) + self._current_url = self._page.url + except Exception as exc: # noqa: BLE001 + raise BrowserError("click_at failed: %s" % exc) + result = self._info() + result["ok"] = True + result["x"] = fx + result["y"] = fy + return result + + def live_shot(self, output_filename=None): + """Screenshot to a FIXED live-view file plus a JSON status + sidecar, so a watcher (console, user) can poll one stable + path instead of chasing per-shot filenames. The agent + should call this after every navigate/click/fill while a + live view is wanted. Returns path, bytes, url, title and + a unix timestamp.""" + self._ensure() + name = _validate_basename( + output_filename or "browser-live.png", "output_filename") + path = os.path.join(self.cwd, name) + try: + self._page.screenshot(path=path) + except Exception as exc: # noqa: BLE001 + raise BrowserError("live_shot failed: %s" % exc) + info = self._info() + status = {"image": name, "path": path, + "bytes": os.path.getsize(path), + "url": info["url"], "title": info["title"], + "timestamp": int(time.time())} + status_path = os.path.join(self.cwd, "browser-live.json") + with open(status_path, "w", encoding="utf-8") as fh: + json.dump(status, fh, ensure_ascii=False) + status["status_path"] = status_path + return status + def evaluate(self, expression): if not expression or not str(expression).strip(): raise BrowserError("expression is required") diff --git a/cloud/playwright_mcp.py b/cloud/playwright_mcp.py index 0e08ddb..635b214 100644 --- a/cloud/playwright_mcp.py +++ b/cloud/playwright_mcp.py @@ -109,6 +109,44 @@ _TOOLS = [ "required": ["expression"], }, }, + { + "name": "browser_click_at", + "description": ( + "Click at viewport coordinates (CSS pixels, origin " + "top-left of the 1280x800 viewport). This is the tool " + "a live viewer drives: the user clicks on a screenshot " + "and the coordinates arrive here. Then snapshot to see " + "the result." + ), + "inputSchema": { + "type": "object", + "properties": { + "x": {"type": "number"}, + "y": {"type": "number"}, + }, + "required": ["x", "y"], + }, + }, + { + "name": "browser_live_shot", + "description": ( + "Screenshot to the FIXED live-view file " + "browser-live.png plus a browser-live.json status " + "sidecar (url, title, timestamp), so a watcher can " + "poll one stable path. Call after every navigate / " + "click / fill while a live view is wanted. Returns " + "path, bytes, url, title and timestamp." + ), + "inputSchema": { + "type": "object", + "properties": { + "output_filename": {"type": "string", + "description": "Plain basename " + "to save as. Default: " + "browser-live.png"}, + }, + }, + }, { "name": "browser_screenshot", "description": ( @@ -199,6 +237,11 @@ def handle_request(request, browser=None): elif name == "browser_fill": result = target.fill(args.get("selector"), args.get("text")) + elif name == "browser_click_at": + result = target.click_at(args.get("x"), args.get("y")) + elif name == "browser_live_shot": + result = target.live_shot( + args.get("output_filename")) elif name == "browser_evaluate": result = target.evaluate(args.get("expression")) elif name == "browser_screenshot": diff --git a/cloud/skills/browser-automation/SKILL.md b/cloud/skills/browser-automation/SKILL.md index fb77a70..384c306 100644 --- a/cloud/skills/browser-automation/SKILL.md +++ b/cloud/skills/browser-automation/SKILL.md @@ -17,11 +17,23 @@ description: > `browser_fill`（按选择器填表）、 `browser_evaluate`（在页内跑 JS 表达式，返回字符串，限 4000 字）、 `browser_screenshot`（把当前页存成工作区内的 PNG，返回绝对路径与字节数）、 +`browser_click_at`（按视口坐标点击，坐标系为 1280x800、原点左上角， +是 live 视图里用户点击的落点）、 +`browser_live_shot`（把当前画面存成固定文件 browser-live.png， +附带 browser-live.json 状态：URL、标题、时间戳，供 watcher 轮询）、 `browser_close`（关闭浏览器，可重复调用）。 浏览器是单页的：一次 navigate，之后 snapshot / click / fill / screenshot 都针对同一页；要换页再 navigate。 +## live 视图（第一期） + +有人在看浏览器时，每次 navigate / click / fill / click_at 之后 +都要调一次 `browser_live_shot`，让 browser-live.png 保持最新； +不要只在最后截图一次。用户在 live 图上点的坐标会以 x/y 到达 +`browser_click_at`，点完必须再 snapshot 一次确认变化，再调 +`browser_live_shot` 刷新画面。 + ## 规则 1. 先 navigate，再 snapshot 看实际渲染结果；click / fill 后 diff --git a/cloud/tests/test_playwright.py b/cloud/tests/test_playwright.py index 5af89e7..97a9086 100644 --- a/cloud/tests/test_playwright.py +++ b/cloud/tests/test_playwright.py @@ -26,11 +26,20 @@ class _FakeLocator: return self._text +class _FakeMouse: + def __init__(self, page): + self._page = page + + def click(self, x, y): + self._page.calls.append(("mouse_click", x, y)) + + class _FakePage: def __init__(self): self.url = "https://example.com/" self.calls = [] self.shot_path = None + self.mouse = _FakeMouse(self) def title(self): return "Example Domain" @@ -146,11 +155,43 @@ class BrowserTest(unittest.TestCase): self.assertTrue(self.browser.close()["closed"]) self.assertTrue(self.browser.close()["closed"]) + def test_click_at_and_bounds(self): + self.browser.navigate("https://example.com") + out = self.browser.click_at(100, 200) + self.assertTrue(out["ok"]) + self.assertEqual(out["x"], 100) + self.assertIn(("mouse_click", 100, 200), self.page.calls) + with self.assertRaises(BrowserError): + self.browser.click_at(2000, 10) + with self.assertRaises(BrowserError): + self.browser.click_at("left", 10) + + def test_live_shot_writes_fixed_file_and_status(self): + self.browser.navigate("https://example.com") + out = self.browser.live_shot() + self.assertEqual(out["image"], "browser-live.png") + self.assertTrue(out["bytes"] > 0) + self.assertTrue(os.path.exists(out["path"])) + with open(os.path.join(self._tmp.name, + "browser-live.json")) as fh: + status = json.load(fh) + self.assertEqual(status["image"], "browser-live.png") + self.assertEqual(status["url"], "https://example.com") + class FakeMCPBrowser: def navigate(self, url): return {"url": url, "title": "T", "ok": True} + def click_at(self, x, y): + return {"url": "https://x/", "title": "T", "ok": True, + "x": x, "y": y} + + def live_shot(self, name=None): + return {"image": name or "browser-live.png", + "path": "/w/browser-live.png", "bytes": 456, + "url": "https://x/", "title": "T", "timestamp": 1} + def snapshot(self): return {"url": "https://x/", "title": "T", "text": "hi", "truncated": False} @@ -168,7 +209,8 @@ class MCPTest(unittest.TestCase): for name in ("browser_navigate", "browser_snapshot", "browser_click", "browser_fill", "browser_evaluate", "browser_screenshot", - "browser_close"): + "browser_close", "browser_click_at", + "browser_live_shot"): self.assertIn(name, names) def test_tools_call_dispatches(self):