websearch: task-level allowed_hosts extension for scope=code + JS-heavy extract evaluation (mock-first)

- search(..., allowed_hosts=[...]) extends (never replaces) the 15-domain
  code list for that call only; scope=general + allowed_hosts raises,
  empty/illegal values raise before any HTTP, dot-boundary matching,
  extension values never in error strings or attempted entries.
- Native-filter parity: Tavily/Exa get the effective list, Perplexity
  skips its native filter past its 20-domain cap (post-filter stays
  authoritative), Brave/SearXNG/DuckDuckGo rely on post-filtering.
- MCP web_search gains allowed_hosts passthrough (only when supplied,
  so older search_fn fakes keep working).
- JS-heavy evaluation documented in docs/web-search-mcp.md, mock-first:
  direct fetch is shell-only by construction; provider extract can
  supply rendered text in mock only; live SearXNG in this VM env
  failed (remote closed connection) and paid-backend live effects are
  unvalidated blockers, not claimed.
- Tests: test_code_search_ext (9) + test_js_heavy_extract (5 + 1 live
  skipped); existing websearch/code-search suites still green.

(cherry picked from commit 9c26bbe848438e1f34bcb000700aeb4c284f4f23)

cloud/tests/test_code_search_ext.py

@@ -0,0 +1,220 @@
+"""Task-level code-scope extension (search(..., allowed_hosts=[...])).
+
+All HTTP is mocked; no network, no real keys. Fake key values are
+distinctive so a leak into an error string or result is caught by
+assertion. Extension values are the caller's own input echoed in
+its own result only -- no test here writes them to a log, and the
+code under test never formats them into error strings.
+"""
+
+import json
+import sys
+import unittest
+from pathlib import Path
+
+try:
+    from cloud import websearch as _ws
+    from cloud.websearch import (
+        CODE_SEARCH_DOMAINS, WebSearchError, _is_code_domain,
+        _normalize_allowed_hosts, search)
+    from cloud.websearch_mcp import handle_request
+except ImportError:
+    sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
+    from cloud import websearch as _ws
+    from cloud.websearch import (
+        CODE_SEARCH_DOMAINS, WebSearchError, _is_code_domain,
+        _normalize_allowed_hosts, search)
+    from cloud.websearch_mcp import handle_request
+
+_FAKE_KEYS = {
+    "PERPLEXITY_API_KEY": "pplx-SECRET-111",
+    "EXA_API_KEY": "exa-SECRET-222",
+    "TAVILY_API_KEY": "tvly-SECRET-333",
+    "BRAVE_API_KEY": "brave-SECRET-444",
+}
+
+_EXTRA = "docs.example-project.dev"
+
+
+class FakeHttp:
+    """post_json/get keyed by URL substring, in route order."""
+
+    def __init__(self, routes):
+        self.routes = routes
+        self.seen = []
+
+    def post_json(self, url, headers, payload):
+        self.seen.append(("POST", url, payload))
+        return self._route(url)
+
+    def get(self, url, headers=None):
+        self.seen.append(("GET", url, headers))
+        return self._route(url)
+
+    def _route(self, url):
+        for needle, body, status in self.routes:
+            if needle in url:
+                if status != 200:
+                    raise WebSearchError("HTTP %s" % status)
+                if isinstance(body, dict):
+                    return 200, json.dumps(body).encode(), \
+                        "application/json"
+                return 200, body.encode(), "text/html"
+        raise WebSearchError("HTTP 404")
+
+
+def _tavily_body(urls):
+    return {"results": [{"title": "T %s" % u, "url": u,
+                         "content": "snippet %s" % u} for u in urls]}
+
+
+class ExtensionTestCase(unittest.TestCase):
+    def test_specified_extra_domain_returned_and_native_param_sent(self):
+        http = FakeHttp([
+            ("api.tavily.com", _tavily_body([
+                "https://docs.example-project.dev/guide",
+                "https://github.com/example/project",
+                "https://random-blog.example.com/post",
+            ]), 200),
+        ])
+        out = search("project guide", provider="tavily", scope="code",
+                     allowed_hosts=[_EXTRA],
+                     env=dict(_FAKE_KEYS), http=http)
+        payload = http.seen[0][2]
+        self.assertIn(_EXTRA, payload["include_domains"])
+        self.assertEqual(out["code_domains"],
+                         list(CODE_SEARCH_DOMAINS) + [_EXTRA])
+        self.assertEqual(
+            [r["url"] for r in out["results"]],
+            ["https://docs.example-project.dev/guide",
+             "https://github.com/example/project"])
+        self.assertEqual(out["attempted"][0]["domain_filtered_out"], 1)
+        for secret in _FAKE_KEYS.values():
+            self.assertNotIn(secret, json.dumps(out))
+
+    def test_unspecified_extra_domain_still_filtered_out(self):
+        http = FakeHttp([
+            ("api.tavily.com", _tavily_body([
+                "https://docs.example-project.dev/guide",
+                "https://github.com/example/project",
+            ]), 200),
+        ])
+        out = search("project guide", provider="tavily", scope="code",
+                     env=dict(_FAKE_KEYS), http=http)
+        self.assertEqual(out["code_domains"], list(CODE_SEARCH_DOMAINS))
+        self.assertEqual([r["url"] for r in out["results"]],
+                         ["https://github.com/example/project"])
+
+    def test_dot_boundary_positive_and_negative(self):
+        domains = list(CODE_SEARCH_DOMAINS) + [_EXTRA]
+        self.assertTrue(_is_code_domain(
+            "https://api.docs.example-project.dev/v1", domains))
+        self.assertTrue(_is_code_domain(
+            "https://docs.example-project.dev/", domains))
+        self.assertFalse(_is_code_domain(
+            "https://evildocs.example-project.dev/", domains))
+        self.assertFalse(_is_code_domain(
+            "https://docs.example-project.dev.evil.com/", domains))
+        # Same boundary rule for a builtin domain, unchanged.
+        self.assertFalse(_is_code_domain(
+            "https://evilgithub.com/x", domains))
+        self.assertTrue(_is_code_domain(
+            "https://gist.github.com/x", domains))
+
+    def test_builtin_entries_are_deduped_not_errors(self):
+        self.assertEqual(
+            _normalize_allowed_hosts(["github.com", _EXTRA]), [_EXTRA])
+        self.assertEqual(
+            _normalize_allowed_hosts(["Docs.Example-Project.Dev."]),
+            ["docs.example-project.dev"])
+
+    def test_empty_and_illegal_values_raise_before_http(self):
+        bad_values = [
+            [], [""], ["   "], ["*.example.com"],
+            ["http://example.com"], ["https://example.com/x"],
+            ["example.com:8080"], ["example.com/path"],
+            [".example.com"], ["example.com..dev"], ["exa mple.com"],
+            ["user@example.com"], [123], [None], ["github.com"],
+        ]
+        for bad in bad_values:
+            http = FakeHttp([("api.tavily.com", _tavily_body([]), 200)])
+            with self.assertRaises(ValueError, msg=repr(bad)):
+                search("q", provider="tavily", scope="code",
+                       allowed_hosts=bad, env=dict(_FAKE_KEYS),
+                       http=http)
+            self.assertEqual(http.seen, [],
+                             "invalid %r must fail before any HTTP"
+                             % (bad,))
+        # A bare string is iterable but is not a list of hostnames.
+        with self.assertRaises(ValueError):
+            search("q", provider="tavily", scope="code",
+                   allowed_hosts="example.com",
+                   env=dict(_FAKE_KEYS), http=FakeHttp([]))
+
+    def test_allowed_hosts_with_general_scope_raises_not_ignored(self):
+        http = FakeHttp([("api.tavily.com", _tavily_body(
+            ["https://docs.example-project.dev/guide"]), 200)])
+        with self.assertRaises(ValueError):
+            search("q", provider="tavily", scope="general",
+                   allowed_hosts=[_EXTRA],
+                   env=dict(_FAKE_KEYS), http=http)
+        self.assertEqual(http.seen, [])
+
+    def test_perplexity_over_cap_skips_native_filter_keeps_postfilter(self):
+        extras = ["extra%d.example.com" % i for i in range(6)]
+        self.assertGreater(len(CODE_SEARCH_DOMAINS) + len(extras), 20)
+        http = FakeHttp([
+            ("api.perplexity.ai",
+             {"choices": [{"message": {"content": "answer"}}],
+              "search_results": [
+                  {"title": "P",
+                   "url": "https://extra0.example.com/doc",
+                   "snippet": "extra"},
+                  {"title": "Ad", "url": "https://shop.example.com/x",
+                   "snippet": "ad"}]}, 200),
+        ])
+        out = search("q", provider="perplexity", scope="code",
+                     allowed_hosts=extras,
+                     env=dict(_FAKE_KEYS), http=http)
+        payload = http.seen[0][2]
+        self.assertNotIn("search_domain_filter", payload)
+        self.assertEqual([r["url"] for r in out["results"]],
+                         ["https://extra0.example.com/doc"])
+        self.assertEqual(len(out["code_domains"]), 21)
+
+
+class McpPassthroughTestCase(unittest.TestCase):
+    def test_mcp_forwards_allowed_hosts_when_supplied(self):
+        captured = {}
+
+        def fake_search(query, limit=5, provider="auto", scope="general",
+                        allowed_hosts=None):
+            captured["allowed_hosts"] = allowed_hosts
+            return {"provider": "fake", "results": [], "answer": None,
+                    "attempted": [], "scope": scope}
+
+        resp = handle_request(
+            {"jsonrpc": "2.0", "id": 1, "method": "tools/call",
+             "params": {"name": "web_search",
+                        "arguments": {"query": "hi", "scope": "code",
+                                      "allowed_hosts": [_EXTRA]}}},
+            search_fn=fake_search)
+        self.assertFalse(resp["result"]["isError"])
+        self.assertEqual(captured["allowed_hosts"], [_EXTRA])
+
+    def test_mcp_without_allowed_hosts_keeps_old_fake_working(self):
+        def fake_search(query, limit=5, provider="auto",
+                        scope="general"):
+            return {"provider": "fake", "results": [], "answer": None,
+                    "attempted": [], "scope": scope}
+
+        resp = handle_request(
+            {"jsonrpc": "2.0", "id": 2, "method": "tools/call",
+             "params": {"name": "web_search",
+                        "arguments": {"query": "hi"}}},
+            search_fn=fake_search)
+        self.assertFalse(resp["result"]["isError"])
+
+
+if __name__ == "__main__":
+    unittest.main()

cloud/tests/test_js_heavy_extract.py

@@ -0,0 +1,155 @@
+"""JS-heavy page extraction evaluation, mock-first.
+
+All HTTP is mocked; no network, no real keys. The three page shapes
+below are the mock stand-ins for real JS-heavy pages (a static shell
+whose body is rendered by JS, a shell whose pagination is JS-loaded,
+and a shell whose content lives only in a <script> blob):
+
+- direct fetch (fetch_url) parses static HTML only: <script> and
+  <style> contents are stripped, so anything JS would have rendered
+  is absent from the result -- this is by construction, not a bug;
+- a provider extract stage (fetch_via_provider, mocked here) can
+  return the rendered text when the provider actually renders, and
+  an empty extraction is a failure, never a fabricated page.
+
+Live对照: only the keyless SearXNG route is ever used live, and
+only when a test environment actually provides SEARXNG_URL; this
+file never calls a paid backend live. Set
+WEBSEARCH_EVAL_LIVE_SEARXNG=1 with SEARXNG_URL in the environment
+to enable the live smoke test, otherwise it is skipped.
+"""
+
+import json
+import os
+import sys
+import unittest
+from pathlib import Path
+
+try:
+    from cloud.websearch import (
+        WebSearchError, fetch_best, fetch_url, fetch_via_provider)
+except ImportError:
+    sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
+    from cloud.websearch import (
+        WebSearchError, fetch_best, fetch_url, fetch_via_provider)
+
+_FAKE_KEYS = {
+    "TAVILY_API_KEY": "tvly-SECRET-333",
+    "EXA_API_KEY": "exa-SECRET-222",
+}
+
+# Shape 1: shell HTML only; the article body is JS-rendered.
+_SHELL_HTML = """<html><head><title>Guide</title></head><body>
+<div id="app">Loading…</div>
+<script src="/app.js"></script>
+<script>window.__DATA__ = {"body": "RENDERED-BODY-PARAGRAPH"};</script>
+<a href="https://example.com/guide/page2">Next (static link)</a>
+</body></html>"""
+
+# Shape 2: paginated list; page 2 items arrive only via JS fetch.
+_PAGED_SHELL_HTML = """<html><head><title>List</title></head><body>
+<ul id="items"><li>ITEM-PAGE-1</li></ul>
+<button id="more">Load more</button>
+<script>fetch("/api/items?page=2").then(r => r.json());</script>
+</body></html>"""
+
+
+class FakeHttp:
+    """get/post_json keyed by URL substring, in route order."""
+
+    def __init__(self, routes):
+        self.routes = routes
+        self.seen = []
+
+    def post_json(self, url, headers, payload):
+        self.seen.append(("POST", url, payload))
+        return self._route(url)
+
+    def get(self, url, headers=None):
+        self.seen.append(("GET", url, headers))
+        return self._route(url)
+
+    def _route(self, url):
+        for needle, body, status in self.routes:
+            if needle in url:
+                if status != 200:
+                    raise WebSearchError("HTTP %s" % status)
+                if isinstance(body, dict):
+                    return 200, json.dumps(body).encode(), \
+                        "application/json"
+                return 200, body.encode(), "text/html"
+        raise WebSearchError("HTTP 404")
+
+
+class JsHeavyDirectFetchTestCase(unittest.TestCase):
+    def test_direct_fetch_gets_shell_only_not_js_rendered_body(self):
+        http = FakeHttp([("example.com/guide", _SHELL_HTML, 200)])
+        out = fetch_url("https://example.com/guide", http=http, env={})
+        # Static shell survives: title + placeholder + static link.
+        self.assertEqual(out["title"], "Guide")
+        self.assertIn("Loading", out["text"])
+        self.assertIn("https://example.com/guide/page2", out["links"])
+        # JS-rendered body does not: <script> contents are stripped,
+        # so neither the rendered paragraph nor the data blob leaks
+        # in as text.
+        self.assertNotIn("RENDERED-BODY-PARAGRAPH", out["text"])
+        self.assertNotIn("window.__DATA__", out["text"])
+
+    def test_direct_fetch_gets_page1_only_not_js_paginated_items(self):
+        http = FakeHttp([("example.com/list", _PAGED_SHELL_HTML, 200)])
+        out = fetch_url("https://example.com/list", http=http, env={})
+        self.assertIn("ITEM-PAGE-1", out["text"])
+        self.assertNotIn("ITEM-PAGE-2", out["text"])
+        self.assertNotIn("/api/items?page=2", out["text"])
+
+
+class JsHeavyProviderExtractTestCase(unittest.TestCase):
+    def test_mock_provider_extract_can_supply_rendered_text(self):
+        # Mock stand-in for "provider renders JS": Tavily extract
+        # returns the full rendered article text.
+        http = FakeHttp([
+            ("api.tavily.com/extract",
+             {"results": [{"raw_content": "RENDERED-BODY-PARAGRAPH "
+                                          "plus ITEM-PAGE-2"}]}, 200),
+        ])
+        out = fetch_via_provider("https://example.com/guide",
+                                 env=dict(_FAKE_KEYS), http=http)
+        self.assertEqual(out["source"], "tavily-extract")
+        self.assertIn("RENDERED-BODY-PARAGRAPH", out["text"])
+
+    def test_mock_provider_extract_empty_is_failure_not_fabrication(self):
+        http = FakeHttp([
+            ("api.tavily.com/extract", {"results": []}, 200),
+            ("api.exa.ai/contents", {"results": []}, 200),
+        ])
+        with self.assertRaises(WebSearchError):
+            fetch_via_provider("https://example.com/guide",
+                               env=dict(_FAKE_KEYS), http=http)
+
+    def test_fetch_best_falls_back_to_shell_when_extract_fails(self):
+        http = FakeHttp([
+            ("api.tavily.com/extract", {"results": []}, 200),
+            ("api.exa.ai/contents", {"results": []}, 200),
+            ("example.com/guide", _SHELL_HTML, 200),
+        ])
+        out = fetch_best("https://example.com/guide",
+                         env=dict(_FAKE_KEYS), http=http)
+        self.assertEqual(out["source"], "direct-fetch")
+        self.assertNotIn("RENDERED-BODY-PARAGRAPH", out["text"])
+
+
+@unittest.skipUnless(
+    os.environ.get("WEBSEARCH_EVAL_LIVE_SEARXNG") == "1"
+    and os.environ.get("SEARXNG_URL"),
+    "live SearXNG对照 only when the test env provides SEARXNG_URL")
+class LiveSearxngSmokeTestCase(unittest.TestCase):
+    def test_live_searxng_chain_answers_without_paid_keys(self):
+        from cloud.websearch import search
+        out = search("python pathlib documentation", provider="searxng",
+                     scope="general",
+                     env={"SEARXNG_URL": os.environ["SEARXNG_URL"]})
+        self.assertTrue(out["results"])
+
+
+if __name__ == "__main__":
+    unittest.main()