feat: 浏览器一期:坐标点击 + live 截图
- browser_click_at(x, y):视口坐标点击(1280x800、左上角 原点),越界与非数字拒绝;live 视图里用户在截图上的 点击落点就走这个工具。 - browser_live_shot:固定文件 browser-live.png + 状态 sidecar browser-live.json(URL/标题/时间戳),watcher 轮询一个稳定路径即可;技能要求每次动作后都调一次, 保持画面最新。 - MCP 新增 browser_click_at / browser_live_shot 两个工具; 技能补 live 视图一节;测试 9 项(含 fake mouse、越界、 live 文件与状态读回)。
This commit is contained in:
1 parent
cb6de0e3aa
commit
814fe8f7c2
4 files changed
+151
-1
No files matched your search
@@ -148,6 +148,59 @@ class Browser:
|
|||||||
result["ok"] = True
|
result["ok"] = True
|
||||||
return result
|
return result
|
||||||
|
|
||||||
|
def click_at(self, x, y):
|
||||||
|
"""Click by viewport coordinates (CSS pixels, origin at the
|
||||||
|
top-left of the 1280x800 viewport). This is the tool the
|
||||||
|
console live view drives: the user clicks on a screenshot
|
||||||
|
and the coordinates arrive here. Bounds-checked so a
|
||||||
|
mis-scaled click cannot scroll the page into nowhere."""
|
||||||
|
try:
|
||||||
|
fx = float(x)
|
||||||
|
fy = float(y)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
raise BrowserError("click_at needs numeric x/y, got %r/%r"
|
||||||
|
% (x, y))
|
||||||
|
if not (0 <= fx <= 1280 and 0 <= fy <= 800):
|
||||||
|
raise BrowserError(
|
||||||
|
"click_at out of viewport 1280x800: %s,%s" % (x, y))
|
||||||
|
self._ensure()
|
||||||
|
try:
|
||||||
|
self._page.mouse.click(fx, fy)
|
||||||
|
self._current_url = self._page.url
|
||||||
|
except Exception as exc: # noqa: BLE001
|
||||||
|
raise BrowserError("click_at failed: %s" % exc)
|
||||||
|
result = self._info()
|
||||||
|
result["ok"] = True
|
||||||
|
result["x"] = fx
|
||||||
|
result["y"] = fy
|
||||||
|
return result
|
||||||
|
|
||||||
|
def live_shot(self, output_filename=None):
|
||||||
|
"""Screenshot to a FIXED live-view file plus a JSON status
|
||||||
|
sidecar, so a watcher (console, user) can poll one stable
|
||||||
|
path instead of chasing per-shot filenames. The agent
|
||||||
|
should call this after every navigate/click/fill while a
|
||||||
|
live view is wanted. Returns path, bytes, url, title and
|
||||||
|
a unix timestamp."""
|
||||||
|
self._ensure()
|
||||||
|
name = _validate_basename(
|
||||||
|
output_filename or "browser-live.png", "output_filename")
|
||||||
|
path = os.path.join(self.cwd, name)
|
||||||
|
try:
|
||||||
|
self._page.screenshot(path=path)
|
||||||
|
except Exception as exc: # noqa: BLE001
|
||||||
|
raise BrowserError("live_shot failed: %s" % exc)
|
||||||
|
info = self._info()
|
||||||
|
status = {"image": name, "path": path,
|
||||||
|
"bytes": os.path.getsize(path),
|
||||||
|
"url": info["url"], "title": info["title"],
|
||||||
|
"timestamp": int(time.time())}
|
||||||
|
status_path = os.path.join(self.cwd, "browser-live.json")
|
||||||
|
with open(status_path, "w", encoding="utf-8") as fh:
|
||||||
|
json.dump(status, fh, ensure_ascii=False)
|
||||||
|
status["status_path"] = status_path
|
||||||
|
return status
|
||||||
|
|
||||||
def evaluate(self, expression):
|
def evaluate(self, expression):
|
||||||
if not expression or not str(expression).strip():
|
if not expression or not str(expression).strip():
|
||||||
raise BrowserError("expression is required")
|
raise BrowserError("expression is required")
|
||||||
|
|||||||
@@ -109,6 +109,44 @@ _TOOLS = [
|
|||||||
"required": ["expression"],
|
"required": ["expression"],
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"name": "browser_click_at",
|
||||||
|
"description": (
|
||||||
|
"Click at viewport coordinates (CSS pixels, origin "
|
||||||
|
"top-left of the 1280x800 viewport). This is the tool "
|
||||||
|
"a live viewer drives: the user clicks on a screenshot "
|
||||||
|
"and the coordinates arrive here. Then snapshot to see "
|
||||||
|
"the result."
|
||||||
|
),
|
||||||
|
"inputSchema": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"x": {"type": "number"},
|
||||||
|
"y": {"type": "number"},
|
||||||
|
},
|
||||||
|
"required": ["x", "y"],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "browser_live_shot",
|
||||||
|
"description": (
|
||||||
|
"Screenshot to the FIXED live-view file "
|
||||||
|
"browser-live.png plus a browser-live.json status "
|
||||||
|
"sidecar (url, title, timestamp), so a watcher can "
|
||||||
|
"poll one stable path. Call after every navigate / "
|
||||||
|
"click / fill while a live view is wanted. Returns "
|
||||||
|
"path, bytes, url, title and timestamp."
|
||||||
|
),
|
||||||
|
"inputSchema": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"output_filename": {"type": "string",
|
||||||
|
"description": "Plain basename "
|
||||||
|
"to save as. Default: "
|
||||||
|
"browser-live.png"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"name": "browser_screenshot",
|
"name": "browser_screenshot",
|
||||||
"description": (
|
"description": (
|
||||||
@@ -199,6 +237,11 @@ def handle_request(request, browser=None):
|
|||||||
elif name == "browser_fill":
|
elif name == "browser_fill":
|
||||||
result = target.fill(args.get("selector"),
|
result = target.fill(args.get("selector"),
|
||||||
args.get("text"))
|
args.get("text"))
|
||||||
|
elif name == "browser_click_at":
|
||||||
|
result = target.click_at(args.get("x"), args.get("y"))
|
||||||
|
elif name == "browser_live_shot":
|
||||||
|
result = target.live_shot(
|
||||||
|
args.get("output_filename"))
|
||||||
elif name == "browser_evaluate":
|
elif name == "browser_evaluate":
|
||||||
result = target.evaluate(args.get("expression"))
|
result = target.evaluate(args.get("expression"))
|
||||||
elif name == "browser_screenshot":
|
elif name == "browser_screenshot":
|
||||||
|
|||||||
@@ -17,11 +17,23 @@ description: >
|
|||||||
`browser_fill`(按选择器填表)、
|
`browser_fill`(按选择器填表)、
|
||||||
`browser_evaluate`(在页内跑 JS 表达式,返回字符串,限 4000 字)、
|
`browser_evaluate`(在页内跑 JS 表达式,返回字符串,限 4000 字)、
|
||||||
`browser_screenshot`(把当前页存成工作区内的 PNG,返回绝对路径与字节数)、
|
`browser_screenshot`(把当前页存成工作区内的 PNG,返回绝对路径与字节数)、
|
||||||
|
`browser_click_at`(按视口坐标点击,坐标系为 1280x800、原点左上角,
|
||||||
|
是 live 视图里用户点击的落点)、
|
||||||
|
`browser_live_shot`(把当前画面存成固定文件 browser-live.png,
|
||||||
|
附带 browser-live.json 状态:URL、标题、时间戳,供 watcher 轮询)、
|
||||||
`browser_close`(关闭浏览器,可重复调用)。
|
`browser_close`(关闭浏览器,可重复调用)。
|
||||||
|
|
||||||
浏览器是单页的:一次 navigate,之后 snapshot / click / fill /
|
浏览器是单页的:一次 navigate,之后 snapshot / click / fill /
|
||||||
screenshot 都针对同一页;要换页再 navigate。
|
screenshot 都针对同一页;要换页再 navigate。
|
||||||
|
|
||||||
|
## live 视图(第一期)
|
||||||
|
|
||||||
|
有人在看浏览器时,每次 navigate / click / fill / click_at 之后
|
||||||
|
都要调一次 `browser_live_shot`,让 browser-live.png 保持最新;
|
||||||
|
不要只在最后截图一次。用户在 live 图上点的坐标会以 x/y 到达
|
||||||
|
`browser_click_at`,点完必须再 snapshot 一次确认变化,再调
|
||||||
|
`browser_live_shot` 刷新画面。
|
||||||
|
|
||||||
## 规则
|
## 规则
|
||||||
|
|
||||||
1. 先 navigate,再 snapshot 看实际渲染结果;click / fill 后
|
1. 先 navigate,再 snapshot 看实际渲染结果;click / fill 后
|
||||||
|
|||||||
@@ -26,11 +26,20 @@ class _FakeLocator:
|
|||||||
return self._text
|
return self._text
|
||||||
|
|
||||||
|
|
||||||
|
class _FakeMouse:
|
||||||
|
def __init__(self, page):
|
||||||
|
self._page = page
|
||||||
|
|
||||||
|
def click(self, x, y):
|
||||||
|
self._page.calls.append(("mouse_click", x, y))
|
||||||
|
|
||||||
|
|
||||||
class _FakePage:
|
class _FakePage:
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
self.url = "https://example.com/"
|
self.url = "https://example.com/"
|
||||||
self.calls = []
|
self.calls = []
|
||||||
self.shot_path = None
|
self.shot_path = None
|
||||||
|
self.mouse = _FakeMouse(self)
|
||||||
|
|
||||||
def title(self):
|
def title(self):
|
||||||
return "Example Domain"
|
return "Example Domain"
|
||||||
@@ -146,11 +155,43 @@ class BrowserTest(unittest.TestCase):
|
|||||||
self.assertTrue(self.browser.close()["closed"])
|
self.assertTrue(self.browser.close()["closed"])
|
||||||
self.assertTrue(self.browser.close()["closed"])
|
self.assertTrue(self.browser.close()["closed"])
|
||||||
|
|
||||||
|
def test_click_at_and_bounds(self):
|
||||||
|
self.browser.navigate("https://example.com")
|
||||||
|
out = self.browser.click_at(100, 200)
|
||||||
|
self.assertTrue(out["ok"])
|
||||||
|
self.assertEqual(out["x"], 100)
|
||||||
|
self.assertIn(("mouse_click", 100, 200), self.page.calls)
|
||||||
|
with self.assertRaises(BrowserError):
|
||||||
|
self.browser.click_at(2000, 10)
|
||||||
|
with self.assertRaises(BrowserError):
|
||||||
|
self.browser.click_at("left", 10)
|
||||||
|
|
||||||
|
def test_live_shot_writes_fixed_file_and_status(self):
|
||||||
|
self.browser.navigate("https://example.com")
|
||||||
|
out = self.browser.live_shot()
|
||||||
|
self.assertEqual(out["image"], "browser-live.png")
|
||||||
|
self.assertTrue(out["bytes"] > 0)
|
||||||
|
self.assertTrue(os.path.exists(out["path"]))
|
||||||
|
with open(os.path.join(self._tmp.name,
|
||||||
|
"browser-live.json")) as fh:
|
||||||
|
status = json.load(fh)
|
||||||
|
self.assertEqual(status["image"], "browser-live.png")
|
||||||
|
self.assertEqual(status["url"], "https://example.com")
|
||||||
|
|
||||||
|
|
||||||
class FakeMCPBrowser:
|
class FakeMCPBrowser:
|
||||||
def navigate(self, url):
|
def navigate(self, url):
|
||||||
return {"url": url, "title": "T", "ok": True}
|
return {"url": url, "title": "T", "ok": True}
|
||||||
|
|
||||||
|
def click_at(self, x, y):
|
||||||
|
return {"url": "https://x/", "title": "T", "ok": True,
|
||||||
|
"x": x, "y": y}
|
||||||
|
|
||||||
|
def live_shot(self, name=None):
|
||||||
|
return {"image": name or "browser-live.png",
|
||||||
|
"path": "/w/browser-live.png", "bytes": 456,
|
||||||
|
"url": "https://x/", "title": "T", "timestamp": 1}
|
||||||
|
|
||||||
def snapshot(self):
|
def snapshot(self):
|
||||||
return {"url": "https://x/", "title": "T", "text": "hi",
|
return {"url": "https://x/", "title": "T", "text": "hi",
|
||||||
"truncated": False}
|
"truncated": False}
|
||||||
@@ -168,7 +209,8 @@ class MCPTest(unittest.TestCase):
|
|||||||
for name in ("browser_navigate", "browser_snapshot",
|
for name in ("browser_navigate", "browser_snapshot",
|
||||||
"browser_click", "browser_fill",
|
"browser_click", "browser_fill",
|
||||||
"browser_evaluate", "browser_screenshot",
|
"browser_evaluate", "browser_screenshot",
|
||||||
"browser_close"):
|
"browser_close", "browser_click_at",
|
||||||
|
"browser_live_shot"):
|
||||||
self.assertIn(name, names)
|
self.assertIn(name, names)
|
||||||
|
|
||||||
def test_tools_call_dispatches(self):
|
def test_tools_call_dispatches(self):
|
||||||
|
|||||||
Reference in new issue
Block a user