websearch: add scope=code programming search + provider extract chain
- search(..., scope=code) restricts to 15 programming domains (official docs, GitHub, registries, Stack Overflow, CVE/NVD): native include_domains/search_domain_filter where supported (Tavily, Exa, Perplexity), post-filtering for all providers (Brave, SearXNG, DuckDuckGo have no native filter). - web_fetch now uses fetch_best: Tavily extract, then Exa contents, then direct fetch; both-fail errors name both failures, direct-fetch 403 under the egress whitelist is fail-closed, not bypassed. Default general scope and direct fetch_url behaviour unchanged. - MCP web_search gains scope enum (general/code); docs updated. - Tests all mock HTTP, no real keys.
This commit is contained in:
1 parent
27f48fe1ed
commit
397a46f850
5 files changed
+803
-36
No files matched your search
+26
-6
@@ -20,10 +20,10 @@ import os
|
||||
import sys
|
||||
|
||||
try:
|
||||
from cloud.websearch import WebSearchError, fetch_url, search
|
||||
from cloud.websearch import WebSearchError, fetch_best, search
|
||||
except ImportError: # flat copy in a tenant codex-home
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
from websearch import WebSearchError, fetch_url, search # noqa: E402
|
||||
from websearch import WebSearchError, fetch_best, search # noqa: E402
|
||||
|
||||
SERVER_NAME = "websearch"
|
||||
SERVER_VERSION = "0.1.0"
|
||||
@@ -37,7 +37,14 @@ _TOOLS = [
|
||||
"supplies one. Use web_fetch to read specific pages. "
|
||||
"provider may be 'auto' (walk Perplexity, Exa, Tavily, "
|
||||
"Brave, then keyless SearXNG / DuckDuckGo, falling back "
|
||||
"on failure) or a single provider name to pin it."
|
||||
"on failure) or a single provider name to pin it. "
|
||||
"scope 'code' restricts results to programming sources "
|
||||
"only (official language docs, GitHub, package "
|
||||
"registries, Stack Overflow, CVE/NVD); anything else "
|
||||
"is filtered out, never returned. Default scope "
|
||||
"'general' is unfiltered. This tool does not do "
|
||||
"shopping, social, places, travel, general news, "
|
||||
"weather, finance, or browser checkout."
|
||||
),
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
@@ -48,6 +55,12 @@ _TOOLS = [
|
||||
"description": "Max results, 1-10"},
|
||||
"provider": {"type": "string", "default": "auto",
|
||||
"description": "auto or a provider name"},
|
||||
"scope": {"type": "string", "default": "general",
|
||||
"enum": ["general", "code"],
|
||||
"description": "general: unfiltered web "
|
||||
"search. code: programming "
|
||||
"sources only, see tool "
|
||||
"description."},
|
||||
},
|
||||
"required": ["query"],
|
||||
},
|
||||
@@ -57,7 +70,13 @@ _TOOLS = [
|
||||
"description": (
|
||||
"Fetch one URL and return clean text plus its links, "
|
||||
"truncated head+tail when over char_limit. "
|
||||
"Only http(s) URLs; localhost / private IPs are blocked."
|
||||
"Extraction first: Tavily extract, then Exa contents "
|
||||
"(only backends with a configured key are tried), "
|
||||
"direct fetch last. Under the egress proxy, direct "
|
||||
"fetch of a domain outside the Tinyproxy whitelist "
|
||||
"fails 403 fail-closed; that error is reported, not "
|
||||
"bypassed. Only http(s) URLs; localhost / private IPs "
|
||||
"are blocked. This is not a sandbox boundary."
|
||||
),
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
@@ -83,7 +102,7 @@ def handle_request(request, search_fn=None, fetch_fn=None):
|
||||
None for notifications (no id). Pure function for testability:
|
||||
the stdio loop passes the real search/fetch, tests pass fakes."""
|
||||
search_fn = search_fn or search
|
||||
fetch_fn = fetch_fn or fetch_url
|
||||
fetch_fn = fetch_fn or fetch_best
|
||||
if not isinstance(request, dict):
|
||||
return {"jsonrpc": "2.0", "id": None,
|
||||
"error": _err(-32700, "request is not an object")}
|
||||
@@ -123,7 +142,8 @@ def handle_request(request, search_fn=None, fetch_fn=None):
|
||||
result = search_fn(
|
||||
args.get("query"),
|
||||
limit=args.get("limit", 5),
|
||||
provider=args.get("provider", "auto"))
|
||||
provider=args.get("provider", "auto"),
|
||||
scope=args.get("scope", "general"))
|
||||
elif name == "web_fetch":
|
||||
result = fetch_fn(
|
||||
args.get("url"),
|
||||
|
||||
Reference in new issue
Block a user