websearch: add scope=code programming search + provider extract chain

- search(..., scope=code) restricts to 15 programming domains
  (official docs, GitHub, registries, Stack Overflow, CVE/NVD):
  native include_domains/search_domain_filter where supported
  (Tavily, Exa, Perplexity), post-filtering for all providers
  (Brave, SearXNG, DuckDuckGo have no native filter).
- web_fetch now uses fetch_best: Tavily extract, then Exa
  contents, then direct fetch; both-fail errors name both
  failures, direct-fetch 403 under the egress whitelist is
  fail-closed, not bypassed. Default general scope and direct
  fetch_url behaviour unchanged.
- MCP web_search gains scope enum (general/code); docs updated.
- Tests all mock HTTP, no real keys.
This commit is contained in:
host committed 2026-11-01 08:16:48 +00:00
1 parent 27f48fe1ed
commit 397a46f850
5 files changed
+803 -36

No files matched your search

+26 -6
View File
@@ -20,10 +20,10 @@ import os
import sys
try:
from cloud.websearch import WebSearchError, fetch_url, search
from cloud.websearch import WebSearchError, fetch_best, search
except ImportError: # flat copy in a tenant codex-home
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from websearch import WebSearchError, fetch_url, search # noqa: E402
from websearch import WebSearchError, fetch_best, search # noqa: E402
SERVER_NAME = "websearch"
SERVER_VERSION = "0.1.0"
@@ -37,7 +37,14 @@ _TOOLS = [
"supplies one. Use web_fetch to read specific pages. "
"provider may be 'auto' (walk Perplexity, Exa, Tavily, "
"Brave, then keyless SearXNG / DuckDuckGo, falling back "
"on failure) or a single provider name to pin it."
"on failure) or a single provider name to pin it. "
"scope 'code' restricts results to programming sources "
"only (official language docs, GitHub, package "
"registries, Stack Overflow, CVE/NVD); anything else "
"is filtered out, never returned. Default scope "
"'general' is unfiltered. This tool does not do "
"shopping, social, places, travel, general news, "
"weather, finance, or browser checkout."
),
"inputSchema": {
"type": "object",
@@ -48,6 +55,12 @@ _TOOLS = [
"description": "Max results, 1-10"},
"provider": {"type": "string", "default": "auto",
"description": "auto or a provider name"},
"scope": {"type": "string", "default": "general",
"enum": ["general", "code"],
"description": "general: unfiltered web "
"search. code: programming "
"sources only, see tool "
"description."},
},
"required": ["query"],
},
@@ -57,7 +70,13 @@ _TOOLS = [
"description": (
"Fetch one URL and return clean text plus its links, "
"truncated head+tail when over char_limit. "
"Only http(s) URLs; localhost / private IPs are blocked."
"Extraction first: Tavily extract, then Exa contents "
"(only backends with a configured key are tried), "
"direct fetch last. Under the egress proxy, direct "
"fetch of a domain outside the Tinyproxy whitelist "
"fails 403 fail-closed; that error is reported, not "
"bypassed. Only http(s) URLs; localhost / private IPs "
"are blocked. This is not a sandbox boundary."
),
"inputSchema": {
"type": "object",
@@ -83,7 +102,7 @@ def handle_request(request, search_fn=None, fetch_fn=None):
None for notifications (no id). Pure function for testability:
the stdio loop passes the real search/fetch, tests pass fakes."""
search_fn = search_fn or search
fetch_fn = fetch_fn or fetch_url
fetch_fn = fetch_fn or fetch_best
if not isinstance(request, dict):
return {"jsonrpc": "2.0", "id": None,
"error": _err(-32700, "request is not an object")}
@@ -123,7 +142,8 @@ def handle_request(request, search_fn=None, fetch_fn=None):
result = search_fn(
args.get("query"),
limit=args.get("limit", 5),
provider=args.get("provider", "auto"))
provider=args.get("provider", "auto"),
scope=args.get("scope", "general"))
elif name == "web_fetch":
result = fetch_fn(
args.get("url"),