websearch: task-level allowed_hosts extension for scope=code + JS-heavy extract evaluation (mock-first)

- search(..., allowed_hosts=[...]) extends (never replaces) the 15-domain
  code list for that call only; scope=general + allowed_hosts raises,
  empty/illegal values raise before any HTTP, dot-boundary matching,
  extension values never in error strings or attempted entries.
- Native-filter parity: Tavily/Exa get the effective list, Perplexity
  skips its native filter past its 20-domain cap (post-filter stays
  authoritative), Brave/SearXNG/DuckDuckGo rely on post-filtering.
- MCP web_search gains allowed_hosts passthrough (only when supplied,
  so older search_fn fakes keep working).
- JS-heavy evaluation documented in docs/web-search-mcp.md, mock-first:
  direct fetch is shell-only by construction; provider extract can
  supply rendered text in mock only; live SearXNG in this VM env
  failed (remote closed connection) and paid-backend live effects are
  unvalidated blockers, not claimed.
- Tests: test_code_search_ext (9) + test_js_heavy_extract (5 + 1 live
  skipped); existing websearch/code-search suites still green.

(cherry picked from commit 9c26bbe848438e1f34bcb000700aeb4c284f4f23)

cloud/tests/test_js_heavy_extract.py

@@ -0,0 +1,155 @@
+"""JS-heavy page extraction evaluation, mock-first.
+
+All HTTP is mocked; no network, no real keys. The three page shapes
+below are the mock stand-ins for real JS-heavy pages (a static shell
+whose body is rendered by JS, a shell whose pagination is JS-loaded,
+and a shell whose content lives only in a <script> blob):
+
+- direct fetch (fetch_url) parses static HTML only: <script> and
+  <style> contents are stripped, so anything JS would have rendered
+  is absent from the result -- this is by construction, not a bug;
+- a provider extract stage (fetch_via_provider, mocked here) can
+  return the rendered text when the provider actually renders, and
+  an empty extraction is a failure, never a fabricated page.
+
+Live对照: only the keyless SearXNG route is ever used live, and
+only when a test environment actually provides SEARXNG_URL; this
+file never calls a paid backend live. Set
+WEBSEARCH_EVAL_LIVE_SEARXNG=1 with SEARXNG_URL in the environment
+to enable the live smoke test, otherwise it is skipped.
+"""
+
+import json
+import os
+import sys
+import unittest
+from pathlib import Path
+
+try:
+    from cloud.websearch import (
+        WebSearchError, fetch_best, fetch_url, fetch_via_provider)
+except ImportError:
+    sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
+    from cloud.websearch import (
+        WebSearchError, fetch_best, fetch_url, fetch_via_provider)
+
+_FAKE_KEYS = {
+    "TAVILY_API_KEY": "tvly-SECRET-333",
+    "EXA_API_KEY": "exa-SECRET-222",
+}
+
+# Shape 1: shell HTML only; the article body is JS-rendered.
+_SHELL_HTML = """<html><head><title>Guide</title></head><body>
+<div id="app">Loading…</div>
+<script src="/app.js"></script>
+<script>window.__DATA__ = {"body": "RENDERED-BODY-PARAGRAPH"};</script>
+<a href="https://example.com/guide/page2">Next (static link)</a>
+</body></html>"""
+
+# Shape 2: paginated list; page 2 items arrive only via JS fetch.
+_PAGED_SHELL_HTML = """<html><head><title>List</title></head><body>
+<ul id="items"><li>ITEM-PAGE-1</li></ul>
+<button id="more">Load more</button>
+<script>fetch("/api/items?page=2").then(r => r.json());</script>
+</body></html>"""
+
+
+class FakeHttp:
+    """get/post_json keyed by URL substring, in route order."""
+
+    def __init__(self, routes):
+        self.routes = routes
+        self.seen = []
+
+    def post_json(self, url, headers, payload):
+        self.seen.append(("POST", url, payload))
+        return self._route(url)
+
+    def get(self, url, headers=None):
+        self.seen.append(("GET", url, headers))
+        return self._route(url)
+
+    def _route(self, url):
+        for needle, body, status in self.routes:
+            if needle in url:
+                if status != 200:
+                    raise WebSearchError("HTTP %s" % status)
+                if isinstance(body, dict):
+                    return 200, json.dumps(body).encode(), \
+                        "application/json"
+                return 200, body.encode(), "text/html"
+        raise WebSearchError("HTTP 404")
+
+
+class JsHeavyDirectFetchTestCase(unittest.TestCase):
+    def test_direct_fetch_gets_shell_only_not_js_rendered_body(self):
+        http = FakeHttp([("example.com/guide", _SHELL_HTML, 200)])
+        out = fetch_url("https://example.com/guide", http=http, env={})
+        # Static shell survives: title + placeholder + static link.
+        self.assertEqual(out["title"], "Guide")
+        self.assertIn("Loading", out["text"])
+        self.assertIn("https://example.com/guide/page2", out["links"])
+        # JS-rendered body does not: <script> contents are stripped,
+        # so neither the rendered paragraph nor the data blob leaks
+        # in as text.
+        self.assertNotIn("RENDERED-BODY-PARAGRAPH", out["text"])
+        self.assertNotIn("window.__DATA__", out["text"])
+
+    def test_direct_fetch_gets_page1_only_not_js_paginated_items(self):
+        http = FakeHttp([("example.com/list", _PAGED_SHELL_HTML, 200)])
+        out = fetch_url("https://example.com/list", http=http, env={})
+        self.assertIn("ITEM-PAGE-1", out["text"])
+        self.assertNotIn("ITEM-PAGE-2", out["text"])
+        self.assertNotIn("/api/items?page=2", out["text"])
+
+
+class JsHeavyProviderExtractTestCase(unittest.TestCase):
+    def test_mock_provider_extract_can_supply_rendered_text(self):
+        # Mock stand-in for "provider renders JS": Tavily extract
+        # returns the full rendered article text.
+        http = FakeHttp([
+            ("api.tavily.com/extract",
+             {"results": [{"raw_content": "RENDERED-BODY-PARAGRAPH "
+                                          "plus ITEM-PAGE-2"}]}, 200),
+        ])
+        out = fetch_via_provider("https://example.com/guide",
+                                 env=dict(_FAKE_KEYS), http=http)
+        self.assertEqual(out["source"], "tavily-extract")
+        self.assertIn("RENDERED-BODY-PARAGRAPH", out["text"])
+
+    def test_mock_provider_extract_empty_is_failure_not_fabrication(self):
+        http = FakeHttp([
+            ("api.tavily.com/extract", {"results": []}, 200),
+            ("api.exa.ai/contents", {"results": []}, 200),
+        ])
+        with self.assertRaises(WebSearchError):
+            fetch_via_provider("https://example.com/guide",
+                               env=dict(_FAKE_KEYS), http=http)
+
+    def test_fetch_best_falls_back_to_shell_when_extract_fails(self):
+        http = FakeHttp([
+            ("api.tavily.com/extract", {"results": []}, 200),
+            ("api.exa.ai/contents", {"results": []}, 200),
+            ("example.com/guide", _SHELL_HTML, 200),
+        ])
+        out = fetch_best("https://example.com/guide",
+                         env=dict(_FAKE_KEYS), http=http)
+        self.assertEqual(out["source"], "direct-fetch")
+        self.assertNotIn("RENDERED-BODY-PARAGRAPH", out["text"])
+
+
+@unittest.skipUnless(
+    os.environ.get("WEBSEARCH_EVAL_LIVE_SEARXNG") == "1"
+    and os.environ.get("SEARXNG_URL"),
+    "live SearXNG对照 only when the test env provides SEARXNG_URL")
+class LiveSearxngSmokeTestCase(unittest.TestCase):
+    def test_live_searxng_chain_answers_without_paid_keys(self):
+        from cloud.websearch import search
+        out = search("python pathlib documentation", provider="searxng",
+                     scope="general",
+                     env={"SEARXNG_URL": os.environ["SEARXNG_URL"]})
+        self.assertTrue(out["results"])
+
+
+if __name__ == "__main__":
+    unittest.main()