admin/codex-app-server
1
0 Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

websearch: add scope=code programming search + provider extract chain

Browse Source 
- search(..., scope=code) restricts to 15 programming domains
  (official docs, GitHub, registries, Stack Overflow, CVE/NVD):
  native include_domains/search_domain_filter where supported
  (Tavily, Exa, Perplexity), post-filtering for all providers
  (Brave, SearXNG, DuckDuckGo have no native filter).
- web_fetch now uses fetch_best: Tavily extract, then Exa
  contents, then direct fetch; both-fail errors name both
  failures, direct-fetch 403 under the egress whitelist is
  fail-closed, not bypassed. Default general scope and direct
  fetch_url behaviour unchanged.
- MCP web_search gains scope enum (general/code); docs updated.
- Tests all mock HTTP, no real keys.
This commit is contained in:
host committed 2026-11-01 08:16:48 +00:00
1 parent 27f48fe1ed
commit 397a46f850
5 files changed
+803 -36
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

cloud/websearch.py
+293 -22
View File
@@ -32,6 +32,16 @@ Security notes:
pretend otherwise. It does refuse non-http(s) URLs and literal
localhost / private-IP URLs, which stops the most trivial SSRF
accidents, but it is not a sandbox.
Code scope (``search(..., scope="code")``):
- Only CODE_SEARCH_DOMAINS results survive, via native provider
filters where they exist (Perplexity search_domain_filter,
Exa/Tavily include_domains) plus post-filtering for every provider.
Default scope="general" is unchanged.
- Fetch chain (``fetch_best``, the web_fetch tool path): Tavily
extract, then Exa contents, then direct ``fetch_url``. Direct
fetch 403ing under the egress whitelist is fail-closed and is
reported, never bypassed.
"""
import html as _html
@@ -47,12 +57,15 @@ from html.parser import HTMLParser
__all__ = [
"DEFAULT_CHAIN",
"CODE_SEARCH_DOMAINS",
"WebSearchError",
"SearchResult",
"available_providers",
"resolve_provider_chain",
"search",
"fetch_url",
"fetch_via_provider",
"fetch_best",
"ensure_codex_home_websearch",
"DEFAULT_FETCH_CHAR_LIMIT",
]
@@ -60,6 +73,54 @@ __all__ = [
DEFAULT_CHAIN = ("perplexity", "exa", "tavily", "brave", "searxng",
"duckduckgo")
DEFAULT_FETCH_CHAR_LIMIT = 15000
# Programming-only search scope (``search(..., scope="code")``).
# Fixed allowlist, deliberately closed: a result is kept only when its
# host is one of these domains or a subdomain of one (dot-boundary
# match, so evilgithub.com never matches github.com). Release notes
# and changelogs are in scope ONLY when hosted on a listed domain
# (e.g. a project's GitHub releases page); there is no generic
# "project official domain" detector, and inventing one would silently
# reopen the scope to arbitrary sites. The list has 15 entries, under
# Perplexity's search_domain_filter cap of 20.
CODE_SEARCH_DOMAINS = (
# Official language / platform docs.
"docs.python.org", "nodejs.org", "go.dev", "pkg.go.dev",
"docs.rs", "doc.rust-lang.org", "developer.mozilla.org",
# Code hosting (githubusercontent.com is GitHub's raw host).
"github.com", "githubusercontent.com",
# Package registries.
"pypi.org", "npmjs.com", "crates.io",
# Q&A and security advisories.
"stackoverflow.com", "nvd.nist.gov", "cve.org",
)
def _is_code_domain(url):
"""True when url's host is in (or under) CODE_SEARCH_DOMAINS."""
try:
host = (urllib.parse.urlparse(url).hostname or "").lower()
except Exception:
return False
host = host.rstrip(".")
for domain in CODE_SEARCH_DOMAINS:
if host == domain or host.endswith("." + domain):
return True
return False
def _normalize_scope(scope):
"""Map scope to "general" (default) or "code"; reject anything else.
Unknown scopes raise before any HTTP happens, so a typo can never
silently degrade to an unscoped search.
"""
if scope is None or str(scope).strip() in ("", "general", "all"):
return "general"
if str(scope).strip() == "code":
return "code"
raise ValueError("unknown search scope: %r (expected 'general' "
"or 'code')" % (scope,))
_USER_AGENT = "codex-websearch-mcp/0.1 (+codex-app-server)"
_KEY_ENV = {
@@ -175,16 +236,22 @@ def _clamp_limit(limit, default=5, lo=1, hi=10):
# ------------------------------------------------------------ providers
def _provider_perplexity(query, limit, env, http):
def _provider_perplexity(query, limit, env, http, domains=None):
# Sonar is a synthesis API: one OpenAI-shaped chat call returns a
# written, cited answer. We surface both the answer and the
# underlying sources, so web_search keeps its "answer + citations"
# shape instead of degrading to a link list.
# Native domain filter: top-level search_domain_filter (Sonar chat
# completions; allowlist mode, max 20 domains -- CODE_SEARCH_DOMAINS
# has 15, so it fits whole).
payload = {"model": "sonar",
"messages": [{"role": "user", "content": query}]}
if domains:
payload["search_domain_filter"] = list(domains)
_status, body, _ctype = http.post_json(
"https://api.perplexity.ai/chat/completions",
_default_headers(env, "perplexity"),
{"model": "sonar",
"messages": [{"role": "user", "content": query}]})
payload)
data = json.loads(body.decode("utf-8", errors="replace"))
choices = data.get("choices") or []
message = (choices[0].get("message") or {}) if choices else {}
@@ -202,13 +269,17 @@ def _provider_perplexity(query, limit, env, http):
return results, answer
def _provider_exa(query, limit, env, http):
def _provider_exa(query, limit, env, http, domains=None):
payload = {"query": query, "numResults": limit, "type": "auto",
"contents": {"text": {"maxCharacters": 2000},
"highlights": True}}
if domains:
# Native domain filter.
payload["include_domains"] = list(domains)
_status, body, _ctype = http.post_json(
"https://api.exa.ai/search",
_default_headers(env, "exa"),
{"query": query, "numResults": limit, "type": "auto",
"contents": {"text": {"maxCharacters": 2000},
"highlights": True}})
payload)
data = json.loads(body.decode("utf-8", errors="replace"))
results = []
for item in data.get("results") or []:
@@ -226,13 +297,17 @@ def _provider_exa(query, limit, env, http):
return results, None
def _provider_tavily(query, limit, env, http):
def _provider_tavily(query, limit, env, http, domains=None):
payload = {"query": query, "search_depth": "basic",
"max_results": limit, "include_answer": False,
"include_raw_content": False}
if domains:
# Native domain filter (max 300 domains).
payload["include_domains"] = list(domains)
_status, body, _ctype = http.post_json(
"https://api.tavily.com/search",
_default_headers(env, "tavily"),
{"query": query, "search_depth": "basic",
"max_results": limit, "include_answer": False,
"include_raw_content": False})
payload)
data = json.loads(body.decode("utf-8", errors="replace"))
results = []
for item in data.get("results") or []:
@@ -245,7 +320,12 @@ def _provider_tavily(query, limit, env, http):
return results, data.get("answer") or None
def _provider_brave(query, limit, env, http):
def _provider_brave(query, limit, env, http, domains=None):
# No native include_domains on Brave's web search endpoint (its
# site: query operator is not used for a 15-domain allowlist:
# multi-domain site: semantics are unproven, and a wrong rewrite
# would silently change the query). Code-scope filtering for
# Brave happens after the call, in search().
url = ("https://api.search.brave.com/res/v1/web/search?"
+ urllib.parse.urlencode(
{"q": query, "count": limit, "extra_snippets": "true"}))
@@ -263,7 +343,10 @@ def _provider_brave(query, limit, env, http):
return results, None
def _provider_searxng(query, limit, env, http):
def _provider_searxng(query, limit, env, http, domains=None):
# No native include_domains is passed: SearXNG's site: handling
# depends on the pinned engines and is not relied on. Code-scope
# filtering happens after the call, in search().
base = (env.get("SEARXNG_URL") or "").rstrip("/")
if not base:
raise WebSearchError("searxng not configured (SEARXNG_URL unset)")
@@ -352,7 +435,9 @@ def _unwrap_ddg(href):
return href
def _provider_duckduckgo(query, limit, env, http):
def _provider_duckduckgo(query, limit, env, http, domains=None):
# No native domain filter; code-scope filtering happens after
# the call, in search().
url = ("https://html.duckduckgo.com/html/?"
+ urllib.parse.urlencode({"q": query}))
_status, body, _ctype = http.get(
@@ -416,8 +501,21 @@ def resolve_provider_chain(provider="auto", env=None):
return names
def search(query, limit=5, provider="auto", env=None, http=None):
"""Search the web. Returns {provider, results, answer, attempted}.
def search(query, limit=5, provider="auto", env=None, http=None,
scope="general"):
"""Search the web. Returns {provider, results, answer, attempted,
scope} (plus code_domains when scope="code").
``scope="code"`` restricts results to CODE_SEARCH_DOMAINS:
providers with a native domain filter (Perplexity
search_domain_filter, Exa/Tavily include_domains) get the list in
the request, and every provider's results are post-filtered to
the same list afterwards, so backends without a native filter
(Brave, SearXNG, DuckDuckGo) are scoped too. A provider whose
results are all filtered out counts as "no results" and the chain
continues; the filter count is recorded on its attempted entry.
Default scope="general" behaves exactly as before (no domain
filter, no post-filtering).
Raises WebSearchError (with .attempted) when every provider in
the chain fails or returns no results. Error strings never contain
@@ -425,6 +523,8 @@ def search(query, limit=5, provider="auto", env=None, http=None):
"""
if not query or not str(query).strip():
raise ValueError("query must be a non-empty string")
scope = _normalize_scope(scope)
domains = list(CODE_SEARCH_DOMAINS) if scope == "code" else None
env = env if env is not None else os.environ
http = http or HttpClient(
timeout=int(env.get("WEBSEARCH_TIMEOUT", "20") or 20))
@@ -439,7 +539,7 @@ def search(query, limit=5, provider="auto", env=None, http=None):
continue
try:
results, answer = _PROVIDERS[name](
str(query), limit, env, http)
str(query), limit, env, http, domains=domains)
except Exception as exc: # noqa: BLE001 - chain must continue
attempted.append({"provider": name, "ok": False,
"error": _http_error_message(exc)})
@@ -448,11 +548,29 @@ def search(query, limit=5, provider="auto", env=None, http=None):
attempted.append({"provider": name, "ok": False,
"error": "no results"})
continue
attempted.append({"provider": name, "ok": True, "error": None})
return {"provider": name,
"results": [dict(r) for r in results[:limit]],
"answer": answer,
"attempted": attempted}
if scope == "code":
kept = [r for r in results if _is_code_domain(r.get("url"))]
if not kept:
attempted.append(
{"provider": name, "ok": False,
"error": "code-domain filter removed all %d results"
% len(results)})
continue
entry = {"provider": name, "ok": True, "error": None}
if len(kept) < len(results):
entry["domain_filtered_out"] = len(results) - len(kept)
attempted.append(entry)
results = kept
else:
attempted.append({"provider": name, "ok": True, "error": None})
out = {"provider": name,
"results": [dict(r) for r in results[:limit]],
"answer": answer,
"attempted": attempted,
"scope": scope}
if scope == "code":
out["code_domains"] = domains
return out
raise WebSearchError(
"all search providers failed: %s"
% "; ".join("%s (%s)" % (a["provider"], a["error"])
@@ -578,6 +696,159 @@ def fetch_url(url, char_limit=DEFAULT_FETCH_CHAR_LIMIT, env=None,
"char_limit": char_limit}
# ------------------------------------------------- provider extract
def _extract_tavily(url, char_limit, env, http):
"""Tavily extract: POST /extract, raw_content is the page text."""
_status, body, _ctype = http.post_json(
"https://api.tavily.com/extract",
_default_headers(env, "tavily"),
{"urls": [url], "extract_depth": "basic", "format": "text"})
data = json.loads(body.decode("utf-8", errors="replace"))
items = data.get("results") or []
for item in items:
if isinstance(item, dict):
text = item.get("raw_content") or item.get("content") or ""
if text:
return text
failed = data.get("failed_results") or []
if failed:
raise WebSearchError("tavily extract failed for url")
raise WebSearchError("tavily extract returned no content")
def _extract_exa(url, char_limit, env, http):
"""Exa contents: POST /contents with ids=[url], full text view."""
# Our own single-page budget, not a claimed API cap: the fetch
# truncation below still applies, this just keeps the request
# from asking for an unbounded page.
budget = min(char_limit, 100000)
_status, body, _ctype = http.post_json(
"https://api.exa.ai/contents",
_default_headers(env, "exa"),
{"ids": [url], "text": {"maxCharacters": budget}})
data = json.loads(body.decode("utf-8", errors="replace"))
items = data.get("results") or data.get("contents") or []
for item in items:
if not isinstance(item, dict):
continue
text = item.get("text") or ""
if not text and item.get("highlights"):
text = " ".join(str(h) for h in item["highlights"][:3])
if text:
return text
raise WebSearchError("exa contents returned no content")
def _fetch_result(url, text, char_limit, source):
text, truncated = _truncate(text, char_limit)
return {"url": str(url), "title": "", "text": text,
"links": [], "truncated": truncated,
"char_limit": char_limit, "source": source}
def fetch_via_provider(url, char_limit=DEFAULT_FETCH_CHAR_LIMIT,
env=None, http=None):
"""Extract one URL through a search provider's own extraction
endpoint, never by fetching the target directly.
Order: Tavily extract, then Exa contents; only backends with a
configured key are tried. Raises WebSearchError (with .attempted)
when no extract provider is configured or every tried one fails.
No result is fabricated: an empty extraction is a failure, not
an empty page. The localhost / private-IP refusal of fetch_url
applies here too, before any provider call.
"""
if not url or not str(url).strip():
raise ValueError("url must be a non-empty string")
env = env if env is not None else os.environ
if env.get("WEBSEARCH_ALLOW_PRIVATE_URLS") != "1":
blocked = _is_blocked_fetch_url(str(url))
if blocked:
raise WebSearchError(blocked)
http = http or HttpClient(
timeout=int(env.get("WEBSEARCH_TIMEOUT", "20") or 20))
try:
char_limit = int(char_limit)
except (TypeError, ValueError):
char_limit = DEFAULT_FETCH_CHAR_LIMIT
char_limit = max(2000, min(500000, char_limit))
attempted = []
extractors = (("tavily", _extract_tavily),
("exa", _extract_exa))
tried = False
for name, fn in extractors:
if name in _KEY_ENV and not env.get(_KEY_ENV[name]):
attempted.append({"provider": name, "ok": False,
"error": "missing API key (%s unset)"
% _KEY_ENV[name]})
continue
tried = True
try:
text = fn(str(url), char_limit, env, http)
except Exception as exc: # noqa: BLE001 - try next extractor
attempted.append({"provider": name, "ok": False,
"error": _http_error_message(exc)})
continue
attempted.append({"provider": name, "ok": True, "error": None})
out = _fetch_result(url, text, char_limit,
"tavily-extract" if name == "tavily"
else "exa-contents")
out["attempted"] = attempted
return out
if not tried:
raise WebSearchError(
"no extract provider configured (set TAVILY_API_KEY "
"and/or EXA_API_KEY); direct fetch is the fallback, "
"see fetch_best",
attempted=attempted)
raise WebSearchError(
"all extract providers failed: %s"
% "; ".join("%s (%s)" % (a["provider"], a["error"])
for a in attempted),
attempted=attempted)
def fetch_best(url, char_limit=DEFAULT_FETCH_CHAR_LIMIT, env=None,
http=None):
"""Fetch one URL: provider extraction first, direct fetch last.
This is what the web_fetch MCP tool calls. When no extract
provider is configured it goes straight to direct fetch (same
behaviour as the old web_fetch); when extraction fails it falls
back to direct fetch; when both fail it raises one error naming
both failures, never a fabricated page. Direct fetch under the
egress proxy 403s for target domains not on the Tinyproxy
whitelist -- that is fail-closed, not a reason to bypass the
proxy, and the 403 surfaces in the error. The localhost /
private-IP refusal is checked before anything else.
"""
if not url or not str(url).strip():
raise ValueError("url must be a non-empty string")
env = env if env is not None else os.environ
if env.get("WEBSEARCH_ALLOW_PRIVATE_URLS") != "1":
blocked = _is_blocked_fetch_url(str(url))
if blocked:
raise WebSearchError(blocked)
extract_attempted = []
try:
return fetch_via_provider(url, char_limit=char_limit,
env=env, http=http)
except WebSearchError as exc:
extract_attempted = list(exc.attempted or [])
extract_error = str(exc)
try:
out = fetch_url(url, char_limit=char_limit, env=env, http=http)
except WebSearchError as exc:
raise WebSearchError(
"provider extraction failed (%s) and direct fetch failed "
"(%s)" % (extract_error, exc))
out = dict(out)
out["source"] = "direct-fetch"
out["extract_attempted"] = extract_attempted
return out
# ------------------------------------------------------- codex-home glue
def ensure_codex_home_websearch(codex_home_dir, command_args=None,