websearch: task-level allowed_hosts extension for scope=code + JS-heavy extract evaluation (mock-first)

- search(..., allowed_hosts=[...]) extends (never replaces) the 15-domain
  code list for that call only; scope=general + allowed_hosts raises,
  empty/illegal values raise before any HTTP, dot-boundary matching,
  extension values never in error strings or attempted entries.
- Native-filter parity: Tavily/Exa get the effective list, Perplexity
  skips its native filter past its 20-domain cap (post-filter stays
  authoritative), Brave/SearXNG/DuckDuckGo rely on post-filtering.
- MCP web_search gains allowed_hosts passthrough (only when supplied,
  so older search_fn fakes keep working).
- JS-heavy evaluation documented in docs/web-search-mcp.md, mock-first:
  direct fetch is shell-only by construction; provider extract can
  supply rendered text in mock only; live SearXNG in this VM env
  failed (remote closed connection) and paid-backend live effects are
  unvalidated blockers, not claimed.
- Tests: test_code_search_ext (9) + test_js_heavy_extract (5 + 1 live
  skipped); existing websearch/code-search suites still green.

(cherry picked from commit 9c26bbe848438e1f34bcb000700aeb4c284f4f23)

cloud/websearch.py

@@ -96,19 +96,83 @@ CODE_SEARCH_DOMAINS = (
 )
 
 
-def _is_code_domain(url):
-    """True when url's host is in (or under) CODE_SEARCH_DOMAINS."""
+def _is_code_domain(url, domains=None):
+    """True when url's host is in (or under) ``domains``.
+
+    Defaults to CODE_SEARCH_DOMAINS. Matching is dot-boundary:
+    the host must equal a listed domain or end with "." + domain,
+    so evilgithub.com never matches github.com and
+    github.com.evil.com never matches either.
+    """
+    if domains is None:
+        domains = CODE_SEARCH_DOMAINS
     try:
         host = (urllib.parse.urlparse(url).hostname or "").lower()
     except Exception:
         return False
     host = host.rstrip(".")
-    for domain in CODE_SEARCH_DOMAINS:
+    for domain in domains:
         if host == domain or host.endswith("." + domain):
             return True
     return False
 
 
+def _normalize_allowed_hosts(allowed_hosts):
+    """Validate a task-level code-scope extension list.
+
+    Returns the normalised extra domains (lowercased, trailing dot
+    stripped, builtin duplicates dropped, order preserved). Anything
+    else raises ValueError before any HTTP happens -- never silently
+    degrading to the builtin list or to an unscoped search:
+
+    - None is not valid here; callers pass it through only by
+      leaving ``search(..., allowed_hosts=None)`` unset.
+    - a bare string is rejected (it is iterable, so accepting it
+      would silently treat "example.com" as a list of characters);
+    - an empty list is rejected (empty means "no extension", which
+      is spelled by omitting the argument, not by passing []);
+    - each entry must be a bare hostname: no scheme, port, path,
+      userinfo, wildcard, whitespace, leading dot or empty label.
+    Extension values are never written to logs: this function only
+    validates, and search() never formats the list into an error
+    string or an attempted entry (the effective list is echoed
+    back to the caller in its own result's ``code_domains`` only).
+    """
+    if isinstance(allowed_hosts, str) or not isinstance(
+            allowed_hosts, (list, tuple)):
+        raise ValueError(
+            "allowed_hosts must be a list of hostnames, not %r"
+            % (type(allowed_hosts).__name__,))
+    if not allowed_hosts:
+        raise ValueError(
+            "allowed_hosts must be non-empty; omit the argument "
+            "for no extension")
+    seen = set()
+    out = []
+    for entry in allowed_hosts:
+        if not isinstance(entry, str):
+            raise ValueError(
+                "allowed_hosts entries must be strings, not %r"
+                % (type(entry).__name__,))
+        host = entry.strip().lower().rstrip(".")
+        if (not host or host.startswith(".") or ".." in host
+                or any(c in host for c in "/:?#@[] \t")
+                or "*" in host or host in CODE_SEARCH_DOMAINS):
+            if host in CODE_SEARCH_DOMAINS:
+                continue
+            raise ValueError(
+                "invalid allowed_hosts entry: %r (expected a bare "
+                "hostname such as 'example.com')" % (entry,))
+        if host not in seen:
+            seen.add(host)
+            out.append(host)
+    if not out:
+        raise ValueError(
+            "allowed_hosts adds no new domain beyond the builtin "
+            "list; omit the argument for no extension")
+    return out
+
+
 def _normalize_scope(scope):
     """Map scope to "general" (default) or "code"; reject anything else.
 
@@ -243,11 +307,19 @@ def _provider_perplexity(query, limit, env, http, domains=None):
     # shape instead of degrading to a link list.
     # Native domain filter: top-level search_domain_filter (Sonar chat
     # completions; allowlist mode, max 20 domains -- CODE_SEARCH_DOMAINS
-    # has 15, so it fits whole).
+    # has 15, so the builtin list fits whole; see the over-cap
+    # handling below for task-level extensions).
     payload = {"model": "sonar",
                "messages": [{"role": "user", "content": query}]}
     if domains:
-        payload["search_domain_filter"] = list(domains)
+        # search_domain_filter caps at 20 domains. The builtin list
+        # is 15, but a task-level allowed_hosts extension can push
+        # the effective list past 20; sending an over-cap filter
+        # would fail the provider call, so in that case the native
+        # filter is skipped and search()'s post-filter -- which
+        # always applies -- remains the authoritative scope.
+        if len(domains) <= 20:
+            payload["search_domain_filter"] = list(domains)
     _status, body, _ctype = http.post_json(
         "https://api.perplexity.ai/chat/completions",
         _default_headers(env, "perplexity"),
@@ -502,7 +574,7 @@ def resolve_provider_chain(provider="auto", env=None):
 
 
 def search(query, limit=5, provider="auto", env=None, http=None,
-           scope="general"):
+           scope="general", allowed_hosts=None):
     """Search the web. Returns {provider, results, answer, attempted,
     scope} (plus code_domains when scope="code").
 
@@ -517,6 +589,20 @@ def search(query, limit=5, provider="auto", env=None, http=None,
     Default scope="general" behaves exactly as before (no domain
     filter, no post-filtering).
 
+    ``allowed_hosts`` is a task-level, explicit extension of the
+    code list only -- it is NOT a general "search any site" switch.
+    It is honoured only with scope="code"; passing it with
+    scope="general" raises ValueError instead of being silently
+    ignored, and omitting it leaves behaviour exactly as before
+    (builtin 15 domains only). When given, the effective list is
+    builtin + validated extras, and the same native-filter /
+    post-filter split applies: native-capable backends receive the
+    effective list (Perplexity skips its native filter when the
+    effective list exceeds its 20-domain cap, relying on the
+    post-filter), and backends without a native filter are scoped
+    by post-filtering alone. Extension values are never formatted
+    into error strings or attempted entries.
+
     Raises WebSearchError (with .attempted) when every provider in
     the chain fails or returns no results. Error strings never contain
     API keys; see the module docstring.
@@ -524,7 +610,17 @@ def search(query, limit=5, provider="auto", env=None, http=None,
     if not query or not str(query).strip():
         raise ValueError("query must be a non-empty string")
     scope = _normalize_scope(scope)
-    domains = list(CODE_SEARCH_DOMAINS) if scope == "code" else None
+    if allowed_hosts is not None and scope != "code":
+        raise ValueError(
+            "allowed_hosts is only valid with scope=\"code\" "
+            "(got scope=%r); it is an extension of the code-domain "
+            "list, not a general search override" % (scope,))
+    if scope == "code":
+        domains = list(CODE_SEARCH_DOMAINS)
+        if allowed_hosts is not None:
+            domains.extend(_normalize_allowed_hosts(allowed_hosts))
+    else:
+        domains = None
     env = env if env is not None else os.environ
     http = http or HttpClient(
         timeout=int(env.get("WEBSEARCH_TIMEOUT", "20") or 20))
@@ -549,7 +645,8 @@ def search(query, limit=5, provider="auto", env=None, http=None,
                               "error": "no results"})
             continue
         if scope == "code":
-            kept = [r for r in results if _is_code_domain(r.get("url"))]
+            kept = [r for r in results
+                    if _is_code_domain(r.get("url"), domains)]
             if not kept:
                 attempted.append(
                     {"provider": name, "ok": False,