codex-app-server/cloud/websearch.py

"""Web search + fetch for codex tasks, following the oh-my-pi pattern.

Zero third-party dependencies; Python standard library only.

Design notes (what is copied from oh-my-pi, and what is not):
- Copied: one unified ``web_search`` tool, a ``SearchProvider`` abstraction,
  and an ``auto`` chain that walks providers in order and falls back to
  the next one on failure. Pinning a single provider by name disables
  fallback, exactly like oh-my-pi's pinned provider.
- Copied: a two-step workflow -- ``web_search`` finds candidate URLs,
  ``web_fetch`` reads a specific page -- and site-agnostic extraction
  that returns clean text plus a link list, not raw HTML.
- NOT copied: oh-my-pi's 23 providers. v1 ships the five that matter
  for a codex task: Perplexity (synthesised answer + citations),
  Exa (semantic search + highlights), Tavily (research + extract),
  Brave (fast, independent index) and two keyless fallbacks
  (SearXNG, DuckDuckGo). Model-native OAuth search providers
  (Gemini / Anthropic / Codex / xAI) are deliberately excluded: inside
  a codex task they add no provider the task does not already have.
- NOT copied: oh-my-pi's TypeScript / Rust extraction stack. Fetch
  here is a small stdlib HTML parser: script/style stripped, text
  collapsed, long pages truncated head+tail like Hermes' deterministic
  budget (default 15000 chars, 75% head / 25% tail, ``[TRUNCATED]``).
- FROZEN default chain (paid freeze): the default ``auto`` chain
  contains only the free backends, SearXNG then DuckDuckGo, even
  when paid keys happen to be set -- a SearXNG failure never falls
  back to a paid backend, so default use can never incur paid cost.
  The four paid backends (Perplexity, Exa, Tavily, Brave) keep their
  code, tests and docs, and still run when explicitly pinned
  (provider="tavily" etc.), which is a deliberate override, not a
  default. Only WEBSEARCH_ENABLE_PAID=1 (exact value "1", default
  off) restores the old default chain; unfreezing is an explicit act
  and may incur paid cost, never a silent default.

Security notes:
- API keys come only from the process environment, never from
  arguments, config files, or this module's return values. Error
  strings contain provider names and HTTP statuses only; they never
  format headers or key values, and the chain test asserts this.
- ``web_fetch`` is not a security boundary: a codex task running
  danger-full-access can already curl anything, so fetch does not
  pretend otherwise. It does refuse non-http(s) URLs and literal
  localhost / private-IP URLs, which stops the most trivial SSRF
  accidents, but it is not a sandbox.

Code scope (``search(..., scope="code")``):
- Only CODE_SEARCH_DOMAINS results survive, via native provider
  filters where they exist (Perplexity search_domain_filter,
  Exa/Tavily include_domains) plus post-filtering for every provider.
  Default scope="general" is unchanged.
- Fetch chain (``fetch_best``, the web_fetch tool path): Tavily
  extract, then Exa contents, then direct ``fetch_url``. Direct
  fetch 403ing under the egress whitelist is fail-closed and is
  reported, never bypassed.
"""

import html as _html
import ipaddress
import json
import os
import socket
import urllib.error
import urllib.parse
import urllib.request
from html.parser import HTMLParser

__all__ = [
    "DEFAULT_CHAIN",
    "DEFAULT_FREE_CHAIN",
    "PAID_PROVIDERS",
    "paid_search_enabled",
    "CODE_SEARCH_DOMAINS",
    "WebSearchError",
    "SearchResult",
    "available_providers",
    "resolve_provider_chain",
    "search",
    "fetch_url",
    "fetch_via_provider",
    "fetch_best",
    "ensure_codex_home_websearch",
    "DEFAULT_FETCH_CHAR_LIMIT",
]

DEFAULT_CHAIN = ("perplexity", "exa", "tavily", "brave", "searxng",
                 "duckduckgo")
# Paid backends, frozen out of the default auto chain (see the module
# docstring). DEFAULT_CHAIN keeps the legacy full order so the
# unfrozen chain is exactly the old one; DEFAULT_FREE_CHAIN is what
# default auto actually walks while frozen.
PAID_PROVIDERS = ("perplexity", "exa", "tavily", "brave")
DEFAULT_FREE_CHAIN = ("searxng", "duckduckgo")
DEFAULT_FETCH_CHAR_LIMIT = 15000

# Programming-only search scope (``search(..., scope="code")``).
# Fixed allowlist, deliberately closed: a result is kept only when its
# host is one of these domains or a subdomain of one (dot-boundary
# match, so evilgithub.com never matches github.com). Release notes
# and changelogs are in scope ONLY when hosted on a listed domain
# (e.g. a project's GitHub releases page); there is no generic
# "project official domain" detector, and inventing one would silently
# reopen the scope to arbitrary sites. The list has 15 entries, under
# Perplexity's search_domain_filter cap of 20.
CODE_SEARCH_DOMAINS = (
    # Official language / platform docs.
    "docs.python.org", "nodejs.org", "go.dev", "pkg.go.dev",
    "docs.rs", "doc.rust-lang.org", "developer.mozilla.org",
    # Code hosting (githubusercontent.com is GitHub's raw host).
    "github.com", "githubusercontent.com",
    # Package registries.
    "pypi.org", "npmjs.com", "crates.io",
    # Q&A and security advisories.
    "stackoverflow.com", "nvd.nist.gov", "cve.org",
)


def _is_code_domain(url, domains=None):
    """True when url's host is in (or under) ``domains``.

    Defaults to CODE_SEARCH_DOMAINS. Matching is dot-boundary:
    the host must equal a listed domain or end with "." + domain,
    so evilgithub.com never matches github.com and
    github.com.evil.com never matches either.
    """
    if domains is None:
        domains = CODE_SEARCH_DOMAINS
    try:
        host = (urllib.parse.urlparse(url).hostname or "").lower()
    except Exception:
        return False
    host = host.rstrip(".")
    for domain in domains:
        if host == domain or host.endswith("." + domain):
            return True
    return False


def _normalize_allowed_hosts(allowed_hosts):
    """Validate a task-level code-scope extension list.

    Returns the normalised extra domains (lowercased, trailing dot
    stripped, builtin duplicates dropped, order preserved). Anything
    else raises ValueError before any HTTP happens -- never silently
    degrading to the builtin list or to an unscoped search:

    - None is not valid here; callers pass it through only by
      leaving ``search(..., allowed_hosts=None)`` unset.
    - a bare string is rejected (it is iterable, so accepting it
      would silently treat "example.com" as a list of characters);
    - an empty list is rejected (empty means "no extension", which
      is spelled by omitting the argument, not by passing []);
    - each entry must be a bare hostname: no scheme, port, path,
      userinfo, wildcard, whitespace, leading dot or empty label.
    Extension values are never written to logs: this function only
    validates, and search() never formats the list into an error
    string or an attempted entry (the effective list is echoed
    back to the caller in its own result's ``code_domains`` only).
    """
    if isinstance(allowed_hosts, str) or not isinstance(
            allowed_hosts, (list, tuple)):
        raise ValueError(
            "allowed_hosts must be a list of hostnames, not %r"
            % (type(allowed_hosts).__name__,))
    if not allowed_hosts:
        raise ValueError(
            "allowed_hosts must be non-empty; omit the argument "
            "for no extension")
    seen = set()
    out = []
    for entry in allowed_hosts:
        if not isinstance(entry, str):
            raise ValueError(
                "allowed_hosts entries must be strings, not %r"
                % (type(entry).__name__,))
        host = entry.strip().lower().rstrip(".")
        if (not host or host.startswith(".") or ".." in host
                or any(c in host for c in "/:?#@[] \t")
                or "*" in host or host in CODE_SEARCH_DOMAINS):
            if host in CODE_SEARCH_DOMAINS:
                continue
            raise ValueError(
                "invalid allowed_hosts entry: %r (expected a bare "
                "hostname such as 'example.com')" % (entry,))
        if host not in seen:
            seen.add(host)
            out.append(host)
    if not out:
        raise ValueError(
            "allowed_hosts adds no new domain beyond the builtin "
            "list; omit the argument for no extension")
    return out


def _normalize_scope(scope):
    """Map scope to "general" (default) or "code"; reject anything else.

    Unknown scopes raise before any HTTP happens, so a typo can never
    silently degrade to an unscoped search.
    """
    if scope is None or str(scope).strip() in ("", "general", "all"):
        return "general"
    if str(scope).strip() == "code":
        return "code"
    raise ValueError("unknown search scope: %r (expected 'general' "
                     "or 'code')" % (scope,))
_USER_AGENT = "codex-websearch-mcp/0.1 (+codex-app-server)"

_KEY_ENV = {
    "perplexity": "PERPLEXITY_API_KEY",
    "exa": "EXA_API_KEY",
    "tavily": "TAVILY_API_KEY",
    "brave": "BRAVE_API_KEY",
}
_FREE_PROVIDERS = ("searxng", "duckduckgo")


class WebSearchError(Exception):
    """All providers in the chain failed (or the pinned one did)."""

    def __init__(self, message, attempted=None):
        super().__init__(message)
        self.attempted = list(attempted or [])


class SearchResult(dict):
    """A single search hit: title, url, snippet, optional score."""

    def __init__(self, title, url, snippet="", score=None):
        super().__init__(title=title or "", url=url or "",
                         snippet=snippet or "")
        if score is not None:
            self["score"] = score


# ---------------------------------------------------------------- HTTP

class HttpClient:
    """Thin urllib wrapper, injectable so tests never touch the net."""

    def __init__(self, timeout=20):
        self.timeout = timeout

    def post_json(self, url, headers, payload):
        data = json.dumps(payload).encode("utf-8")
        req = urllib.request.Request(
            url, data=data,
            headers=dict(headers, **{"Content-Type": "application/json"}),
            method="POST")
        return self._read(req)

    def get(self, url, headers=None):
        req = urllib.request.Request(
            url, headers=headers or {}, method="GET")
        return self._read(req)

    def _read(self, req):
        try:
            with urllib.request.urlopen(req, timeout=self.timeout) as resp:
                body = resp.read()
                return resp.status, body, resp.headers.get_content_type()
        except urllib.error.HTTPError as exc:
            # Never include headers (keys live there) in the error.
            body = exc.read()[:500]
            raise WebSearchError(
                "HTTP %s" % exc.code) from _ProviderHttpError(
                    exc.code, body)
        except (socket.timeout, TimeoutError) as exc:
            # A timeout is a timeout, never "no results": the caller
            # (search's attempted entry, named by provider) must be
            # able to tell which stage timed out, and a timed-out
            # SearXNG must not be rewritten as an empty answer.
            # HTTPError is a URLError subclass, so this branch must
            # stay after the HTTPError branch above.
            raise WebSearchError(
                "timeout after %ss waiting for HTTP response"
                % self.timeout) from exc
        except urllib.error.URLError as exc:
            reason = getattr(exc, "reason", None)
            if isinstance(reason, (socket.timeout, TimeoutError)):
                raise WebSearchError(
                    "timeout after %ss waiting for HTTP response"
                    % self.timeout) from exc
            # DNS / connection failures keep their own message;
            # they are not timeouts and must not be relabelled.
            raise WebSearchError(
                "URL error: %s" % reason if reason else "URL error"
                ) from exc

    def get_text(self, url, headers=None):
        status, body, _ctype = self.get(url, headers)
        charset = "utf-8"
        try:
            text = body.decode(charset, errors="replace")
        except Exception:
            text = body.decode("utf-8", errors="replace")
        return status, text


class _ProviderHttpError(Exception):
    def __init__(self, status, body):
        super().__init__("HTTP %s" % status)
        self.status = status
        self.body = body


def _http_error_message(exc):
    """Render a provider failure without ever leaking credentials."""
    if isinstance(exc, _ProviderHttpError):
        return "HTTP %s" % exc.status
    if isinstance(exc, WebSearchError):
        return str(exc)
    return "%s: %s" % (type(exc).__name__, exc)


def _default_headers(env, provider):
    key_env = _KEY_ENV.get(provider)
    key = env.get(key_env) if key_env else None
    if provider == "exa":
        return {"x-api-key": key, "User-Agent": _USER_AGENT}
    if provider == "tavily":
        return {"Authorization": "Bearer %s" % key,
                "User-Agent": _USER_AGENT}
    if provider == "brave":
        return {"X-Subscription-Token": key, "Accept": "application/json",
                "User-Agent": _USER_AGENT}
    if provider == "perplexity":
        return {"Authorization": "Bearer %s" % key,
                "User-Agent": _USER_AGENT}
    return {"User-Agent": _USER_AGENT}


def _clamp_limit(limit, default=5, lo=1, hi=10):
    try:
        value = int(limit)
    except (TypeError, ValueError):
        return default
    return max(lo, min(hi, value))


# ------------------------------------------------------------ providers

def _provider_perplexity(query, limit, env, http, domains=None):
    # Sonar is a synthesis API: one OpenAI-shaped chat call returns a
    # written, cited answer. We surface both the answer and the
    # underlying sources, so web_search keeps its "answer + citations"
    # shape instead of degrading to a link list.
    # Native domain filter: top-level search_domain_filter (Sonar chat
    # completions; allowlist mode, max 20 domains -- CODE_SEARCH_DOMAINS
    # has 15, so the builtin list fits whole; see the over-cap
    # handling below for task-level extensions).
    payload = {"model": "sonar",
               "messages": [{"role": "user", "content": query}]}
    if domains:
        # search_domain_filter caps at 20 domains. The builtin list
        # is 15, but a task-level allowed_hosts extension can push
        # the effective list past 20; sending an over-cap filter
        # would fail the provider call, so in that case the native
        # filter is skipped and search()'s post-filter -- which
        # always applies -- remains the authoritative scope.
        if len(domains) <= 20:
            payload["search_domain_filter"] = list(domains)
    _status, body, _ctype = http.post_json(
        "https://api.perplexity.ai/chat/completions",
        _default_headers(env, "perplexity"),
        payload)
    data = json.loads(body.decode("utf-8", errors="replace"))
    choices = data.get("choices") or []
    message = (choices[0].get("message") or {}) if choices else {}
    answer = message.get("content") or None
    results = []
    for item in data.get("search_results") or []:
        if isinstance(item, dict) and item.get("url"):
            results.append(SearchResult(item.get("title") or item["url"],
                                        item["url"],
                                        item.get("snippet") or ""))
    if not results:
        for url in data.get("citations") or []:
            if isinstance(url, str) and url.startswith("http"):
                results.append(SearchResult(url, url, ""))
    return results, answer


def _provider_exa(query, limit, env, http, domains=None):
    payload = {"query": query, "numResults": limit, "type": "auto",
               "contents": {"text": {"maxCharacters": 2000},
                            "highlights": True}}
    if domains:
        # Native domain filter.
        payload["include_domains"] = list(domains)
    _status, body, _ctype = http.post_json(
        "https://api.exa.ai/search",
        _default_headers(env, "exa"),
        payload)
    data = json.loads(body.decode("utf-8", errors="replace"))
    results = []
    for item in data.get("results") or []:
        if not isinstance(item, dict) or not item.get("url"):
            continue
        highlights = item.get("highlights") or []
        snippet = item.get("text") or ""
        if highlights and not snippet:
            snippet = " ".join(str(h) for h in highlights[:3])
        elif highlights:
            snippet = snippet[:2000]
        results.append(SearchResult(item.get("title") or item["url"],
                                    item["url"], snippet,
                                    item.get("score")))
    return results, None


def _provider_tavily(query, limit, env, http, domains=None):
    payload = {"query": query, "search_depth": "basic",
               "max_results": limit, "include_answer": False,
               "include_raw_content": False}
    if domains:
        # Native domain filter (max 300 domains).
        payload["include_domains"] = list(domains)
    _status, body, _ctype = http.post_json(
        "https://api.tavily.com/search",
        _default_headers(env, "tavily"),
        payload)
    data = json.loads(body.decode("utf-8", errors="replace"))
    results = []
    for item in data.get("results") or []:
        if not isinstance(item, dict) or not item.get("url"):
            continue
        results.append(SearchResult(item.get("title") or item["url"],
                                    item["url"],
                                    item.get("content") or "",
                                    item.get("score")))
    return results, data.get("answer") or None


def _provider_brave(query, limit, env, http, domains=None):
    # No native include_domains on Brave's web search endpoint (its
    # site: query operator is not used for a 15-domain allowlist:
    # multi-domain site: semantics are unproven, and a wrong rewrite
    # would silently change the query). Code-scope filtering for
    # Brave happens after the call, in search().
    url = ("https://api.search.brave.com/res/v1/web/search?"
           + urllib.parse.urlencode(
               {"q": query, "count": limit, "extra_snippets": "true"}))
    _status, body, _ctype = http.get(url, _default_headers(env, "brave"))
    data = json.loads(body.decode("utf-8", errors="replace"))
    results = []
    for item in (data.get("web") or {}).get("results") or []:
        if not isinstance(item, dict) or not item.get("url"):
            continue
        parts = [item.get("description") or ""]
        parts.extend(str(s) for s in (item.get("extra_snippets") or []))
        snippet = " ".join(p for p in parts if p)
        results.append(SearchResult(item.get("title") or item["url"],
                                    item["url"], snippet))
    return results, None


def _provider_searxng(query, limit, env, http, domains=None):
    # No native include_domains is passed: SearXNG's site: handling
    # depends on the pinned engines and is not relied on. Code-scope
    # filtering happens after the call, in search().
    base = (env.get("SEARXNG_URL") or "").rstrip("/")
    if not base:
        raise WebSearchError("searxng not configured (SEARXNG_URL unset)")
    # Engines are pinned, not left to SearXNG's default "general"
    # category. Measured 2026-10-30 on the production host:
    # default general search took 20.0s in a Docker task container
    # (wikipedia / wikidata / brave / duckduckgo / google engines
    # all timing out, only bing + 360search returning), which hits
    # the MCP's 20s timeout and then falls through to DuckDuckGo,
    # which itself times out via the egress proxy -- so the whole
    # chain fails even though SearXNG can answer. Pinning to the
    # two proven-fast engines cuts the same query to 0.47s via
    # the same proxy. SEARXNG_ENGINES overrides the pin; an empty
    # value restores the old unpinned behaviour.
    params = {"q": query, "format": "json"}
    engines = env.get("SEARXNG_ENGINES", "bing,360search")
    if engines:
        params["engines"] = engines
    url = base + "/search?" + urllib.parse.urlencode(params)
    _status, body, _ctype = http.get(url, _default_headers(env, "searxng"))
    data = json.loads(body.decode("utf-8", errors="replace"))
    results = []
    for item in data.get("results") or []:
        if not isinstance(item, dict) or not item.get("url"):
            continue
        results.append(SearchResult(item.get("title") or item["url"],
                                    item["url"],
                                    item.get("content") or ""))
    return results[:limit], None


class _DuckDuckGoParser(HTMLParser):
    """Best-effort parser for DuckDuckGo's HTML results page.

    Only the last-resort free path uses this; it is deliberately
    tolerant of layout drift (missing classes just yield no results,
    which moves the chain on, it never crashes the tool).
    """

    def __init__(self):
        super().__init__()
        self.results = []
        self._current = None
        self._in_title = False
        self._in_snippet = False

    def handle_starttag(self, tag, attrs):
        attrs = dict(attrs)
        cls = attrs.get("class") or ""
        if tag == "a" and "result__a" in cls:
            href = attrs.get("href") or ""
            self._current = {"title": "", "url": _unwrap_ddg(href),
                             "snippet": ""}
            self._in_title = True
        elif tag == "a" and "result__snippet" in cls:
            self._in_snippet = True

    def handle_endtag(self, tag):
        if tag == "a" and self._in_title:
            self._in_title = False
            if self._current and self._current["url"]:
                self.results.append(self._current)
            self._current = None
        if tag == "a" and self._in_snippet:
            self._in_snippet = False

    def handle_data(self, data):
        if self._in_title and self._current is not None:
            self._current["title"] += data
        elif self._in_snippet and self.results:
            self.results[-1]["snippet"] += data


def _unwrap_ddg(href):
    """DuckDuckGo wraps result links as //duckduckgo.com/l/?uddg=<url>."""
    if not href:
        return ""
    parsed = urllib.parse.urlparse(href)
    if "duckduckgo.com" in (parsed.netloc or ""):
        target = urllib.parse.parse_qs(parsed.query).get("uddg")
        if target:
            return target[0]
        return ""
    if href.startswith("//"):
        return "https:" + href
    return href


def _provider_duckduckgo(query, limit, env, http, domains=None):
    # No native domain filter; code-scope filtering happens after
    # the call, in search().
    url = ("https://html.duckduckgo.com/html/?"
           + urllib.parse.urlencode({"q": query}))
    _status, body, _ctype = http.get(
        url, {"User-Agent": _USER_AGENT})
    parser = _DuckDuckGoParser()
    parser.feed(body.decode("utf-8", errors="replace"))
    results = []
    for item in parser.results[:limit]:
        results.append(SearchResult(item["title"].strip(),
                                    item["url"],
                                    item["snippet"].strip()))
    return results, None


_PROVIDERS = {
    "perplexity": _provider_perplexity,
    "exa": _provider_exa,
    "tavily": _provider_tavily,
    "brave": _provider_brave,
    "searxng": _provider_searxng,
    "duckduckgo": _provider_duckduckgo,
}


def paid_search_enabled(env=None):
    """True only when the paid-unfreeze switch is explicitly on.

    The switch is WEBSEARCH_ENABLE_PAID, and only the exact value
    "1" counts (same strict convention as WEBSEARCH_ALLOW_PRIVATE_URLS
    elsewhere in this module): any other value, including "true" or
    "yes", leaves the paid backends frozen out of default auto.
    Explicit pinning is unaffected by this switch either way.
    """
    env = env if env is not None else os.environ
    return env.get("WEBSEARCH_ENABLE_PAID") == "1"


def available_providers(env=None):
    """Providers usable right now, under the paid-freeze rule.

    Frozen (default): only the free chain -- SearXNG when its URL
    is set, plus DuckDuckGo -- even when paid keys are present.
    Unfrozen (WEBSEARCH_ENABLE_PAID=1): the legacy full chain,
    keyed backends only when their key is set, as before.
    """
    env = env if env is not None else os.environ
    chain = (DEFAULT_CHAIN if paid_search_enabled(env)
             else DEFAULT_FREE_CHAIN)
    out = []
    for name in chain:
        if name in _FREE_PROVIDERS:
            if name == "searxng" and not env.get("SEARXNG_URL"):
                continue
            out.append(name)
        elif env.get(_KEY_ENV[name]):
            out.append(name)
    return out


def resolve_provider_chain(provider="auto", env=None):
    """Resolve the chain oh-my-pi style: auto walks, pinned does not.

    ``provider`` may be "auto" (default chain, filtered to what is
    configured -- frozen by default to the free chain, SearXNG then
    DuckDuckGo, unless WEBSEARCH_ENABLE_PAID=1 restores the legacy
    full chain), a single name (that provider only, even if it fails
    -- pinning a paid backend bypasses the freeze, deliberately), or
    a comma-separated list (that order only). Unknown names raise
    ValueError before any HTTP happens.
    """
    env = env if env is not None else os.environ
    if provider is None or str(provider).strip() in ("", "auto"):
        chain = list(available_providers(env))
        if not chain:
            chain = list(_FREE_PROVIDERS)
            # duckduckgo needs no configuration; searxng without a URL
            # will fail fast with a clear message, which is better
            # than pretending no provider exists at all.
            chain = ["duckduckgo"]
        return chain
    names = [p.strip() for p in str(provider).split(",") if p.strip()]
    for name in names:
        if name not in _PROVIDERS:
            raise ValueError("unknown search provider: %r" % name)
    return names


def search(query, limit=5, provider="auto", env=None, http=None,
           scope="general", allowed_hosts=None):
    """Search the web. Returns {provider, results, answer, attempted,
    scope} (plus code_domains when scope="code").

    ``scope="code"`` restricts results to CODE_SEARCH_DOMAINS:
    providers with a native domain filter (Perplexity
    search_domain_filter, Exa/Tavily include_domains) get the list in
    the request, and every provider's results are post-filtered to
    the same list afterwards, so backends without a native filter
    (Brave, SearXNG, DuckDuckGo) are scoped too. A provider whose
    results are all filtered out counts as "no results" and the chain
    continues; the filter count is recorded on its attempted entry.
    Default scope="general" behaves exactly as before (no domain
    filter, no post-filtering).

    ``allowed_hosts`` is a task-level, explicit extension of the
    code list only -- it is NOT a general "search any site" switch.
    It is honoured only with scope="code"; passing it with
    scope="general" raises ValueError instead of being silently
    ignored, and omitting it leaves behaviour exactly as before
    (builtin 15 domains only). When given, the effective list is
    builtin + validated extras, and the same native-filter /
    post-filter split applies: native-capable backends receive the
    effective list (Perplexity skips its native filter when the
    effective list exceeds its 20-domain cap, relying on the
    post-filter), and backends without a native filter are scoped
    by post-filtering alone. Extension values are never formatted
    into error strings or attempted entries.

    Raises WebSearchError (with .attempted) when every provider in
    the chain fails or returns no results. Error strings never contain
    API keys; see the module docstring.
    """
    if not query or not str(query).strip():
        raise ValueError("query must be a non-empty string")
    scope = _normalize_scope(scope)
    if allowed_hosts is not None and scope != "code":
        raise ValueError(
            "allowed_hosts is only valid with scope=\"code\" "
            "(got scope=%r); it is an extension of the code-domain "
            "list, not a general search override" % (scope,))
    if scope == "code":
        domains = list(CODE_SEARCH_DOMAINS)
        if allowed_hosts is not None:
            domains.extend(_normalize_allowed_hosts(allowed_hosts))
    else:
        domains = None
    env = env if env is not None else os.environ
    http = http or HttpClient(
        timeout=int(env.get("WEBSEARCH_TIMEOUT", "20") or 20))
    limit = _clamp_limit(limit)
    chain = resolve_provider_chain(provider, env)
    attempted = []
    for name in chain:
        if name in _KEY_ENV and not env.get(_KEY_ENV[name]):
            attempted.append({"provider": name, "ok": False,
                              "error": "missing API key (%s unset)"
                                       % _KEY_ENV[name]})
            continue
        try:
            results, answer = _PROVIDERS[name](
                str(query), limit, env, http, domains=domains)
        except Exception as exc:  # noqa: BLE001 - chain must continue
            attempted.append({"provider": name, "ok": False,
                              "error": _http_error_message(exc)})
            continue
        if not results:
            attempted.append({"provider": name, "ok": False,
                              "error": "no results"})
            continue
        if scope == "code":
            kept = [r for r in results
                    if _is_code_domain(r.get("url"), domains)]
            if not kept:
                attempted.append(
                    {"provider": name, "ok": False,
                     "error": "code-domain filter removed all %d results"
                              % len(results)})
                continue
            entry = {"provider": name, "ok": True, "error": None}
            if len(kept) < len(results):
                entry["domain_filtered_out"] = len(results) - len(kept)
            attempted.append(entry)
            results = kept
        else:
            attempted.append({"provider": name, "ok": True, "error": None})
        out = {"provider": name,
               "results": [dict(r) for r in results[:limit]],
               "answer": answer,
               "attempted": attempted,
               "scope": scope}
        if scope == "code":
            out["code_domains"] = domains
        return out
    raise WebSearchError(
        "all search providers failed: %s"
        % "; ".join("%s (%s)" % (a["provider"], a["error"])
                    for a in attempted),
        attempted=attempted)


# ---------------------------------------------------------------- fetch

class _TextExtractor(HTMLParser):
    def __init__(self):
        super().__init__()
        self.title = ""
        self._in_title = False
        self._skip = 0
        self.blocks = []
        self.links = []
        self._current_block = []

    def handle_starttag(self, tag, attrs):
        if tag in ("script", "style", "noscript"):
            self._skip += 1
            return
        if tag == "title":
            self._in_title = True
        if tag == "a":
            href = dict(attrs).get("href") or ""
            if href.startswith("http"):
                self.links.append(href)
        if tag in ("p", "h1", "h2", "h3", "h4", "li", "pre", "blockquote"):
            self._flush()

    def handle_endtag(self, tag):
        if tag in ("script", "style", "noscript"):
            self._skip = max(0, self._skip - 1)
        if tag == "title":
            self._in_title = False
        if tag in ("p", "h1", "h2", "h3", "h4", "li", "pre", "blockquote"):
            self._flush()

    def handle_data(self, data):
        if self._skip:
            return
        if self._in_title:
            self.title += data
            return
        text = data.strip()
        if text:
            self._current_block.append(text)

    def _flush(self):
        if self._current_block:
            self.blocks.append(" ".join(self._current_block))
            self._current_block = []

    def text(self):
        self._flush()
        return "\n\n".join(self.blocks)


def _truncate(text, char_limit):
    if len(text) <= char_limit:
        return text, False
    head = int(char_limit * 0.75)
    tail = char_limit - head
    return (text[:head] + "\n\n[TRUNCATED]\n\n" + text[-tail:]
            if tail > 0 else text[:head] + "\n\n[TRUNCATED]"), True


def _is_blocked_fetch_url(url):
    parsed = urllib.parse.urlparse(url)
    if parsed.scheme not in ("http", "https"):
        return "only http(s) URLs can be fetched"
    host = (parsed.hostname or "").lower()
    if host in ("localhost",):
        return "localhost URLs are blocked"
    try:
        addr = ipaddress.ip_address(host)
    except ValueError:
        return None
    if addr.is_private or addr.is_loopback or addr.is_link_local:
        return "private / loopback URLs are blocked"
    return None


def fetch_url(url, char_limit=DEFAULT_FETCH_CHAR_LIMIT, env=None,
              http=None):
    """Fetch one URL and return clean text + links (truncated)."""
    if not url or not str(url).strip():
        raise ValueError("url must be a non-empty string")
    env = env if env is not None else os.environ
    if env.get("WEBSEARCH_ALLOW_PRIVATE_URLS") != "1":
        blocked = _is_blocked_fetch_url(str(url))
        if blocked:
            raise WebSearchError(blocked)
    http = http or HttpClient(
        timeout=int(env.get("WEBSEARCH_TIMEOUT", "20") or 20))
    try:
        char_limit = int(char_limit)
    except (TypeError, ValueError):
        char_limit = DEFAULT_FETCH_CHAR_LIMIT
    char_limit = max(2000, min(500000, char_limit))
    _status, body, ctype = http.get(str(url), {"User-Agent": _USER_AGENT})
    if ctype and not any(
            token in ctype for token in
            ("text/html", "text/plain", "application/json",
             "application/xml", "text/")):
        raise WebSearchError("unsupported content type: %s" % ctype)
    text_in = body.decode("utf-8", errors="replace")
    if "text/html" in (ctype or "") or "<html" in text_in[:2000].lower():
        parser = _TextExtractor()
        parser.feed(text_in)
        title = _html.unescape(parser.title.strip())
        text = _html.unescape(parser.text())
        links = parser.links
    else:
        title = ""
        text = text_in
        links = []
    text, truncated = _truncate(text, char_limit)
    return {"url": str(url), "title": title, "text": text,
            "links": links[:100], "truncated": truncated,
            "char_limit": char_limit}


# ------------------------------------------------- provider extract

def _extract_tavily(url, char_limit, env, http):
    """Tavily extract: POST /extract, raw_content is the page text."""
    _status, body, _ctype = http.post_json(
        "https://api.tavily.com/extract",
        _default_headers(env, "tavily"),
        {"urls": [url], "extract_depth": "basic", "format": "text"})
    data = json.loads(body.decode("utf-8", errors="replace"))
    items = data.get("results") or []
    for item in items:
        if isinstance(item, dict):
            text = item.get("raw_content") or item.get("content") or ""
            if text:
                return text
    failed = data.get("failed_results") or []
    if failed:
        raise WebSearchError("tavily extract failed for url")
    raise WebSearchError("tavily extract returned no content")


def _extract_exa(url, char_limit, env, http):
    """Exa contents: POST /contents with ids=[url], full text view."""
    # Our own single-page budget, not a claimed API cap: the fetch
    # truncation below still applies, this just keeps the request
    # from asking for an unbounded page.
    budget = min(char_limit, 100000)
    _status, body, _ctype = http.post_json(
        "https://api.exa.ai/contents",
        _default_headers(env, "exa"),
        {"ids": [url], "text": {"maxCharacters": budget}})
    data = json.loads(body.decode("utf-8", errors="replace"))
    items = data.get("results") or data.get("contents") or []
    for item in items:
        if not isinstance(item, dict):
            continue
        text = item.get("text") or ""
        if not text and item.get("highlights"):
            text = " ".join(str(h) for h in item["highlights"][:3])
        if text:
            return text
    raise WebSearchError("exa contents returned no content")


def _fetch_result(url, text, char_limit, source):
    text, truncated = _truncate(text, char_limit)
    return {"url": str(url), "title": "", "text": text,
            "links": [], "truncated": truncated,
            "char_limit": char_limit, "source": source}


def fetch_via_provider(url, char_limit=DEFAULT_FETCH_CHAR_LIMIT,
                       env=None, http=None):
    """Extract one URL through a search provider's own extraction
    endpoint, never by fetching the target directly.

    Order: Tavily extract, then Exa contents; only backends with a
    configured key are tried. Raises WebSearchError (with .attempted)
    when no extract provider is configured or every tried one fails.
    No result is fabricated: an empty extraction is a failure, not
    an empty page. The localhost / private-IP refusal of fetch_url
    applies here too, before any provider call.
    """
    if not url or not str(url).strip():
        raise ValueError("url must be a non-empty string")
    env = env if env is not None else os.environ
    if env.get("WEBSEARCH_ALLOW_PRIVATE_URLS") != "1":
        blocked = _is_blocked_fetch_url(str(url))
        if blocked:
            raise WebSearchError(blocked)
    http = http or HttpClient(
        timeout=int(env.get("WEBSEARCH_TIMEOUT", "20") or 20))
    try:
        char_limit = int(char_limit)
    except (TypeError, ValueError):
        char_limit = DEFAULT_FETCH_CHAR_LIMIT
    char_limit = max(2000, min(500000, char_limit))
    attempted = []
    extractors = (("tavily", _extract_tavily),
                  ("exa", _extract_exa))
    tried = False
    for name, fn in extractors:
        if name in _KEY_ENV and not env.get(_KEY_ENV[name]):
            attempted.append({"provider": name, "ok": False,
                              "error": "missing API key (%s unset)"
                                       % _KEY_ENV[name]})
            continue
        tried = True
        try:
            text = fn(str(url), char_limit, env, http)
        except Exception as exc:  # noqa: BLE001 - try next extractor
            attempted.append({"provider": name, "ok": False,
                              "error": _http_error_message(exc)})
            continue
        attempted.append({"provider": name, "ok": True, "error": None})
        out = _fetch_result(url, text, char_limit,
                            "tavily-extract" if name == "tavily"
                            else "exa-contents")
        out["attempted"] = attempted
        return out
    if not tried:
        raise WebSearchError(
            "no extract provider configured (set TAVILY_API_KEY "
            "and/or EXA_API_KEY); direct fetch is the fallback, "
            "see fetch_best",
            attempted=attempted)
    raise WebSearchError(
        "all extract providers failed: %s"
        % "; ".join("%s (%s)" % (a["provider"], a["error"])
                    for a in attempted),
        attempted=attempted)


def fetch_best(url, char_limit=DEFAULT_FETCH_CHAR_LIMIT, env=None,
               http=None):
    """Fetch one URL: provider extraction first, direct fetch last.

    This is what the web_fetch MCP tool calls. When no extract
    provider is configured it goes straight to direct fetch (same
    behaviour as the old web_fetch); when extraction fails it falls
    back to direct fetch; when both fail it raises one error naming
    both failures, never a fabricated page. Direct fetch under the
    egress proxy 403s for target domains not on the Tinyproxy
    whitelist -- that is fail-closed, not a reason to bypass the
    proxy, and the 403 surfaces in the error. The localhost /
    private-IP refusal is checked before anything else.
    """
    if not url or not str(url).strip():
        raise ValueError("url must be a non-empty string")
    env = env if env is not None else os.environ
    if env.get("WEBSEARCH_ALLOW_PRIVATE_URLS") != "1":
        blocked = _is_blocked_fetch_url(str(url))
        if blocked:
            raise WebSearchError(blocked)
    extract_attempted = []
    try:
        return fetch_via_provider(url, char_limit=char_limit,
                                  env=env, http=http)
    except WebSearchError as exc:
        extract_attempted = list(exc.attempted or [])
        extract_error = str(exc)
    try:
        out = fetch_url(url, char_limit=char_limit, env=env, http=http)
    except WebSearchError as exc:
        raise WebSearchError(
            "provider extraction failed (%s) and direct fetch failed "
            "(%s)" % (extract_error, exc))
    out = dict(out)
    out["source"] = "direct-fetch"
    out["extract_attempted"] = extract_attempted
    return out


# ------------------------------------------------------- codex-home glue

def ensure_codex_home_websearch(codex_home_dir, command_args=None,
                                script_dir=None):
    """Seed websearch MCP files + config into a tenant codex-home.

    Idempotent and non-destructive by design:
    - copies ``websearch.py`` and ``websearch_mcp.py`` only when the
      destination is missing or differs (so a tenant's own edits to
      the script are never silently overwritten -- differing content
      is left alone and reported);
    - appends the ``[mcp_servers.websearch]`` section only when that
      section header is absent; an existing section is never rewritten
      (keys are never written into config.toml by this helper).
    Returns {config_path, script_path, config_changed, script_changed}.
    ``command_args`` defaults to ``python3 <codex-home>/websearch_mcp.py``
    on the host; Docker tenants must pass the container path
    (``/var/lib/codex-app-server/codex-home/websearch_mcp.py``)
    instead, because the same config.toml cannot hold both paths.
    """
    import shutil

    home = os.path.abspath(codex_home_dir)
    os.makedirs(home, exist_ok=True)
    here = os.path.abspath(script_dir or os.path.dirname(__file__))
    script_changed = False
    script_differs = []
    for name in ("websearch.py", "websearch_mcp.py"):
        src = os.path.join(here, name)
        dst = os.path.join(home, name)
        if not os.path.exists(src):
            continue
        if not os.path.exists(dst):
            shutil.copyfile(src, dst)
            script_changed = True
        else:
            with open(src, "rb") as fh:
                src_bytes = fh.read()
            with open(dst, "rb") as fh:
                dst_bytes = fh.read()
            if src_bytes != dst_bytes:
                # Do not clobber a tenant-modified script; report it.
                script_differs.append(name)
                continue
    config_path = os.path.join(home, "config.toml")
    if command_args is None:
        command_args = ["python3",
                        os.path.join(home, "websearch_mcp.py")]
    section = ("\n[mcp_servers.websearch]\n"
               "command = %s\n"
               "args = [%s]\n"
               % (json.dumps(command_args[0]),
                  ", ".join(json.dumps(a) for a in command_args[1:])))
    config_changed = False
    existing = ""
    if os.path.exists(config_path):
        with open(config_path, encoding="utf-8") as fh:
            existing = fh.read()
    if "[mcp_servers.websearch]" not in existing:
        with open(config_path, "a", encoding="utf-8") as fh:
            if existing and not existing.endswith("\n"):
                fh.write("\n")
            fh.write(section)
        config_changed = True
    return {"config_path": config_path,
            "script_path": os.path.join(home, "websearch_mcp.py"),
            "config_changed": config_changed,
            "script_changed": script_changed,
            "script_differs": script_differs}


def ensure_websearch_mcp_env(codex_home_dir, env):
    """Write [mcp_servers.websearch.env] into a tenant config.toml.

    Why this exists (measured 2026-10-30, not guessed): Codex's
    MCP server subprocess does NOT inherit the Codex process env.
    A Docker task whose Codex bash env contained SEARXNG_URL and
    http_proxy still ran web_search with chain ["duckduckgo"]
    only, failing instantly with "Temporary failure in name
    resolution" -- SEARXNG_URL never reached the MCP process, and
    without a proxy the container has no direct DNS. Adding an
    explicit ``env`` table to the MCP config made the same task
    return real SearXNG results. Container env (docker -e) is
    therefore NOT sufficient for MCP tools; the env must live in
    config.toml, where Codex actually reads it for MCP servers.

    Idempotent: an existing env subsection is replaced in place,
    other sections are left untouched, and a missing websearch
    section is created first (command/args only, no env values
    invented). Values are written as TOML basic strings via
    json.dumps (safe for URLs and proxy URLs). Returns True when
    the file was written, False when env is empty or the file
    could not be written (caller logs, never crashes the task).
    """
    if not env:
        return False
    home = os.path.abspath(codex_home_dir)
    config_path = os.path.join(home, "config.toml")
    try:
        existing = ""
        if os.path.exists(config_path):
            with open(config_path, encoding="utf-8") as fh:
                existing = fh.read()
        if "[mcp_servers.websearch]" not in existing:
            existing += ("\n[mcp_servers.websearch]\n"
                         "command = \"python3\"\n"
                         "args = [\"/var/lib/codex-app-server/"
                         "codex-home/websearch_mcp.py\"]\n")
        env_lines = ["[mcp_servers.websearch.env]"]
        for key in sorted(env):
            env_lines.append("%s = %s" % (key, json.dumps(env[key])))
        env_block = "\n".join(env_lines) + "\n"
        lines = existing.splitlines(keepends=True)
        start = end = None
        for i, line in enumerate(lines):
            if line.strip() == "[mcp_servers.websearch.env]":
                start = i
            elif start is not None and line.startswith("["):
                end = i
                break
        if start is not None:
            if end is None:
                end = len(lines)
            lines[start:end] = [env_block]
            new_text = "".join(lines)
        else:
            insert_at = None
            in_section = False
            for i, line in enumerate(lines):
                if line.strip() == "[mcp_servers.websearch]":
                    in_section = True
                elif in_section and line.startswith("["):
                    insert_at = i
                    break
            if insert_at is None:
                insert_at = len(lines)
            lines.insert(insert_at, env_block)
            new_text = "".join(lines)
        if new_text == existing:
            return False
        with open(config_path, "w", encoding="utf-8") as fh:
            fh.write(new_text)
        return True
    except Exception:
        return False