websearch: 默认 auto 付费冻结,SearXNG 优先 + 显式解冻开关

- 默认 auto 只走免费链 searxng -> duckduckgo;即使环境里有
  付费 key 也不调用 Perplexity/Exa/Tavily/Brave,SearXNG
  失败也不回落付费,默认使用零付费花费。
- 付费后端代码/测试/文档全部保留;显式 pin(tavily/exa/
  perplexity/brave)仍照常调用,是有意覆盖不是默认。
- 解冻开关 WEBSEARCH_ENABLE_PAID,仅精确值 "1" 才恢复旧链
  (perplexity/exa/tavily/brave/searxng/duckduckgo),默认关;
  开关经 worker search_env 透传进 MCP config env。
- HttpClient 超时/DNS 失败改报显式错误(timeout after Ns /
  URL error),绝不把超时写成“无结果”。
- SearXNG 默认参数核实:代码无 SEARXNG_URL 硬编码默认值;
  SEARXNG_ENGINES 默认 bing,360search 已在基线;WEBSEARCH_TIMEOUT
  默认 20 秒。主机 env 实测仅 SEARXNG_URL=http://searx.diante.me
  一行,故默认 pin 生效;文档 https 漂移改回 http(主机 nginx
  仅听 80、无 443 vhost,https 在 TLS 层即失败)。
- 旧测试更新原因:test_websearch_mcp 原断言默认 auto 为
  perplexity 优先,与新冻结语义直接冲突,故改为断言冻结后
  的 searxng -> duckduckgo 回退 + 不泄漏 key;旧 perplexity ->
  exa 回退断言移至解冻开关测试(WEBSEARCH_ENABLE_PAID=1 下)。
  未删断言、未加 skip、未放宽门禁。
- 文档另删 WEBSEARCH_PROVIDERS 说法:代码从未实现该变量。
- 未推、未建 PR、未合并、未部署、未改主机 env/nginx/filter/
  证书、未做付费后端 live 验证、未创建生产任务。
This commit is contained in:
host committed 2026-11-01 14:15:54 +00:00
1 parent 0602d06136
commit 3a56f83114
7 files changed
+492 -43

No files matched your search

+27 -7
View File
@@ -67,18 +67,38 @@ def _exa_body():
class ChainTestCase(unittest.TestCase):
def test_fallback_perplexity_to_exa_and_key_not_leaked(self):
def test_default_auto_frozen_falls_back_searxng_to_duckduckgo_and_key_not_leaked(self):
# Paid-freeze update of the old perplexity -> exa fallback
# test: default auto no longer contains any paid backend,
# even when all four paid keys are set, so the fallback
# that still exists by default is searxng -> duckduckgo
# and no paid URL is ever touched. The old perplexity ->
# exa fallback is preserved, but only under
# WEBSEARCH_ENABLE_PAID=1; see test_websearch_paid_freeze.py.
html = (
'<a class="result__a" href="//duckduckgo.com/l/?uddg='
'https%3A%2F%2Fexample.com%2Fddg">DDG Title</a>'
'<a class="result__snippet">ddg snippet</a>')
http = FakeHttp([
("perplexity", {}, 500),
("api.exa.ai", _exa_body(), 200),
("searx", {}, 500),
("duckduckgo", html, 200),
])
env = dict(_FAKE_KEYS)
env["SEARXNG_URL"] = "http://searx.test"
out = search("test query", provider="auto",
env=dict(_FAKE_KEYS), http=http)
self.assertEqual(out["provider"], "exa")
env=env, http=http)
self.assertEqual(out["provider"], "duckduckgo")
self.assertEqual(out["results"][0]["url"],
"https://example.com/exa")
self.assertEqual(out["attempted"][0]["provider"], "perplexity")
"https://example.com/ddg")
self.assertEqual(out["attempted"][0]["provider"], "searxng")
self.assertFalse(out["attempted"][0]["ok"])
self.assertEqual(
[a["provider"] for a in out["attempted"]],
["searxng", "duckduckgo"])
for _method, url, _payload in http.seen:
for paid in ("perplexity", "exa.ai", "tavily",
"search.brave.com"):
self.assertNotIn(paid, url)
blob = json.dumps(out)
for secret in _FAKE_KEYS.values():
self.assertNotIn(secret, blob)
+280
View File
@@ -0,0 +1,280 @@
"""Paid-freeze tests: default auto is free-only, pin still works,
unfreeze is explicit (WEBSEARCH_ENABLE_PAID=1, exact value "1").
No network, no real keys: every HTTP call goes through a FakeHttp,
and the fake key values are deliberately distinctive so a leak into
an error string or a tool result would be caught by assertion.
"""
import json
import os
import socket
import sys
import unittest
import urllib.error
from pathlib import Path
from unittest import mock
try:
from cloud.websearch import (
DEFAULT_CHAIN, DEFAULT_FREE_CHAIN, HttpClient, WebSearchError,
available_providers, paid_search_enabled,
resolve_provider_chain, search)
from cloud.websearch_mcp import handle_request
except ImportError:
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
from cloud.websearch import (
DEFAULT_CHAIN, DEFAULT_FREE_CHAIN, HttpClient, WebSearchError,
available_providers, paid_search_enabled,
resolve_provider_chain, search)
from cloud.websearch_mcp import handle_request
_FAKE_KEYS = {
"PERPLEXITY_API_KEY": "pplx-SECRET-111",
"EXA_API_KEY": "exa-SECRET-222",
"TAVILY_API_KEY": "tvly-SECRET-333",
"BRAVE_API_KEY": "brave-SECRET-444",
}
_DDG_HTML = (
'<a class="result__a" href="//duckduckgo.com/l/?uddg='
'https%3A%2F%2Fexample.com%2Fddg">DDG Title</a>'
'<a class="result__snippet">ddg snippet</a>')
class FakeHttp:
"""post_json/get keyed by URL substring; failures raise like HTTP."""
def __init__(self, routes):
self.routes = routes
self.seen = []
def post_json(self, url, headers, payload):
self.seen.append(("POST", url, payload))
return self._route(url)
def get(self, url, headers=None):
self.seen.append(("GET", url, headers))
return self._route(url)
def _route(self, url):
for needle, body, status in self.routes:
if needle in url:
if status != 200:
raise WebSearchError("HTTP %s" % status)
if isinstance(body, dict):
return 200, json.dumps(body).encode(), "application/json"
return 200, body.encode(), "text/html"
raise WebSearchError("HTTP 404")
def _searxng_body():
return {"results": [
{"title": "SearXNG Hit", "url": "https://example.com/sx",
"content": "searxng snippet"}]}
def _paid_env(**extra):
env = dict(_FAKE_KEYS)
env["SEARXNG_URL"] = "http://searx.test"
env.update(extra)
return env
def _paid_hosts_seen(http):
hosts = []
for _method, url, _payload in http.seen:
for paid in ("perplexity", "exa.ai", "tavily",
"search.brave.com"):
if paid in url:
hosts.append(paid)
return hosts
class FrozenDefaultTestCase(unittest.TestCase):
def test_default_auto_excludes_paid_even_with_all_keys(self):
env = _paid_env()
self.assertEqual(available_providers(env),
["searxng", "duckduckgo"])
self.assertEqual(resolve_provider_chain("auto", env),
["searxng", "duckduckgo"])
http = FakeHttp([("searx", _searxng_body(), 200)])
out = search("q", provider="auto", env=env, http=http)
self.assertEqual(out["provider"], "searxng")
self.assertEqual(_paid_hosts_seen(http), [])
def test_default_auto_without_searxng_url_is_duckduckgo_only(self):
env = dict(_FAKE_KEYS)
self.assertEqual(resolve_provider_chain("auto", env),
["duckduckgo"])
http = FakeHttp([("duckduckgo", _DDG_HTML, 200)])
out = search("q", provider="auto", env=env, http=http)
self.assertEqual(out["provider"], "duckduckgo")
self.assertEqual(_paid_hosts_seen(http), [])
def test_searxng_failure_never_falls_back_to_paid(self):
http = FakeHttp([("searx", {}, 500),
("duckduckgo", "<html>challenge</html>", 200)])
with self.assertRaises(WebSearchError) as ctx:
search("q", provider="auto", env=_paid_env(), http=http)
self.assertEqual(
[a["provider"] for a in ctx.exception.attempted],
["searxng", "duckduckgo"])
self.assertEqual(_paid_hosts_seen(http), [])
# No fabricated result: the error names both free stages.
self.assertIn("searxng", str(ctx.exception))
self.assertIn("duckduckgo", str(ctx.exception))
def test_duckduckgo_fallback_failure_reports_clearly(self):
# DuckDuckGo under the VM/host egress returns a 202-style
# robot-challenge page: the parser yields no results, which
# must surface as an honest "no results" failure for that
# stage, never as a fabricated hit.
http = FakeHttp([("searx", {}, 500),
("duckduckgo", "<html>anomaly</html>", 200)])
with self.assertRaises(WebSearchError) as ctx:
search("q", provider="auto", env=_paid_env(), http=http)
attempted = {a["provider"]: a for a in ctx.exception.attempted}
self.assertEqual(attempted["duckduckgo"]["error"], "no results")
self.assertFalse(attempted["duckduckgo"]["ok"])
for secret in _FAKE_KEYS.values():
self.assertNotIn(secret, str(ctx.exception))
class ExplicitPinTestCase(unittest.TestCase):
def test_explicit_pin_paid_still_works_while_frozen(self):
http = FakeHttp([
("api.tavily.com",
{"results": [{"title": "T", "url": "https://example.com/t",
"content": "tavily content"}]}, 200),
])
out = search("q", provider="tavily", env=_paid_env(), http=http)
self.assertEqual(out["provider"], "tavily")
self.assertEqual(out["results"][0]["snippet"], "tavily content")
def test_explicit_pin_paid_failure_does_not_fall_back(self):
http = FakeHttp([("perplexity", {}, 500),
("api.exa.ai", {"results": [
{"title": "E", "url": "https://example.com/e",
"text": "x"}]}, 200)])
with self.assertRaises(WebSearchError):
search("q", provider="perplexity", env=_paid_env(), http=http)
self.assertEqual(_paid_hosts_seen(http), ["perplexity"])
def test_frozen_default_applies_to_scope_code_too(self):
body = {"results": [
{"title": "GH", "url": "https://github.com/a/b",
"content": "code hit"},
{"title": "Other", "url": "https://example.com/other",
"content": "off-scope"}]}
http = FakeHttp([("searx", body, 200)])
out = search("q", provider="auto", env=_paid_env(),
http=http, scope="code")
self.assertEqual(out["provider"], "searxng")
self.assertEqual(
[r["url"] for r in out["results"]], ["https://github.com/a/b"])
self.assertEqual(_paid_hosts_seen(http), [])
class UnfreezeSwitchTestCase(unittest.TestCase):
def test_switch_off_by_default_and_non_1_values_do_not_unfreeze(self):
self.assertFalse(paid_search_enabled({}))
for value in ("0", "true", "yes", "", "on"):
env = _paid_env(WEBSEARCH_ENABLE_PAID=value)
self.assertFalse(paid_search_enabled(env), value)
self.assertEqual(resolve_provider_chain("auto", env),
["searxng", "duckduckgo"])
def test_switch_on_restores_legacy_chain_and_fallback(self):
env = _paid_env(WEBSEARCH_ENABLE_PAID="1")
self.assertTrue(paid_search_enabled(env))
self.assertEqual(resolve_provider_chain("auto", env),
list(DEFAULT_CHAIN))
# The pre-freeze perplexity -> exa fallback, preserved
# verbatim under the explicit switch (plus the no-leak
# assertion from the old default-chain test).
http = FakeHttp([
("perplexity", {}, 500),
("api.exa.ai", {"results": [
{"title": "Exa Hit", "url": "https://example.com/exa",
"text": "exa snippet", "score": 0.9}]}, 200),
])
out = search("test query", provider="auto", env=env, http=http)
self.assertEqual(out["provider"], "exa")
self.assertEqual(out["results"][0]["url"],
"https://example.com/exa")
self.assertEqual(out["attempted"][0]["provider"], "perplexity")
self.assertFalse(out["attempted"][0]["ok"])
blob = json.dumps(out)
for secret in _FAKE_KEYS.values():
self.assertNotIn(secret, blob)
def test_constants_state_the_contract(self):
self.assertEqual(DEFAULT_FREE_CHAIN, ("searxng", "duckduckgo"))
self.assertEqual(
DEFAULT_CHAIN,
("perplexity", "exa", "tavily", "brave", "searxng",
"duckduckgo"))
class TimeoutClarityTestCase(unittest.TestCase):
def test_http_client_maps_timeout_to_explicit_error(self):
client = HttpClient(timeout=20)
with mock.patch("urllib.request.urlopen",
side_effect=socket.timeout("timed out")):
with self.assertRaises(WebSearchError) as ctx:
client.get("http://searx.test/search?q=x")
self.assertIn("timeout after 20s", str(ctx.exception))
with mock.patch(
"urllib.request.urlopen",
side_effect=urllib.error.URLError(
socket.timeout("timed out"))):
with self.assertRaises(WebSearchError) as ctx:
client.get("http://searx.test/search?q=x")
self.assertIn("timeout after 20s", str(ctx.exception))
def test_http_client_does_not_relabel_dns_failure_as_timeout(self):
client = HttpClient(timeout=20)
with mock.patch(
"urllib.request.urlopen",
side_effect=urllib.error.URLError(
"Temporary failure in name resolution")):
with self.assertRaises(WebSearchError) as ctx:
client.get("http://searx.test/search?q=x")
self.assertNotIn("timeout", str(ctx.exception))
self.assertIn("name resolution", str(ctx.exception))
def test_search_reports_timeout_as_error_not_no_results(self):
class TimeoutHttp(FakeHttp):
def get(self, url, headers=None):
self.seen.append(("GET", url, headers))
if "searx" in url:
raise WebSearchError(
"timeout after 20s waiting for HTTP response")
return self._route(url)
http = TimeoutHttp([("duckduckgo", _DDG_HTML, 200)])
out = search("q", provider="auto", env=_paid_env(), http=http)
self.assertEqual(out["provider"], "duckduckgo")
attempted = {a["provider"]: a for a in out["attempted"]}
self.assertIn("timeout after 20s",
attempted["searxng"]["error"])
self.assertNotEqual(attempted["searxng"]["error"], "no results")
class McpDescriptionTestCase(unittest.TestCase):
def test_tools_list_describes_frozen_default_and_unfreeze(self):
resp = handle_request(
{"jsonrpc": "2.0", "id": 1, "method": "tools/list"})
tool = [t for t in resp["result"]["tools"]
if t["name"] == "web_search"][0]
desc = tool["description"]
self.assertIn("SearXNG", desc)
self.assertIn("DuckDuckGo", desc)
self.assertIn("WEBSEARCH_ENABLE_PAID=1", desc)
self.assertIn("paid", desc.lower())
if __name__ == "__main__":
unittest.main()
+15 -1
View File
@@ -124,10 +124,24 @@ class DockerEnvTestCase(unittest.TestCase):
self.assertIsNone(err)
self.assertEqual(env.get("SEARXNG_URL"), "http://searx.diante.me")
def test_paid_unfreeze_switch_passes_into_container_env(self):
# The freeze switch must reach the MCP process (which reads
# only config.toml env, never worker env); without this
# passthrough, WEBSEARCH_ENABLE_PAID=1 on the host would
# silently leave default auto frozen inside containers.
from unittest import mock
with mock.patch.dict(os.environ,
{"WEBSEARCH_ENABLE_PAID": "1"}):
ok, _, _, env, err = self._docker_context()
self.assertTrue(ok)
self.assertIsNone(err)
self.assertEqual(env.get("WEBSEARCH_ENABLE_PAID"), "1")
def test_no_search_env_means_no_extra_env(self):
from unittest import mock
with mock.patch.dict(os.environ, {}, clear=False):
for k in ("SEARXNG_URL", "PERPLEXITY_API_KEY", "EXA_API_KEY",
for k in ("SEARXNG_URL", "WEBSEARCH_ENABLE_PAID",
"PERPLEXITY_API_KEY", "EXA_API_KEY",
"TAVILY_API_KEY", "BRAVE_API_KEY"):
os.environ.pop(k, None)
ok, _, _, env, err = self._docker_context()
+69 -5
View File
@@ -21,6 +21,16 @@ Design notes (what is copied from oh-my-pi, and what is not):
here is a small stdlib HTML parser: script/style stripped, text
collapsed, long pages truncated head+tail like Hermes' deterministic
budget (default 15000 chars, 75% head / 25% tail, ``[TRUNCATED]``).
- FROZEN default chain (paid freeze): the default ``auto`` chain
contains only the free backends, SearXNG then DuckDuckGo, even
when paid keys happen to be set -- a SearXNG failure never falls
back to a paid backend, so default use can never incur paid cost.
The four paid backends (Perplexity, Exa, Tavily, Brave) keep their
code, tests and docs, and still run when explicitly pinned
(provider="tavily" etc.), which is a deliberate override, not a
default. Only WEBSEARCH_ENABLE_PAID=1 (exact value "1", default
off) restores the old default chain; unfreezing is an explicit act
and may incur paid cost, never a silent default.
Security notes:
- API keys come only from the process environment, never from
@@ -57,6 +67,9 @@ from html.parser import HTMLParser
__all__ = [
"DEFAULT_CHAIN",
"DEFAULT_FREE_CHAIN",
"PAID_PROVIDERS",
"paid_search_enabled",
"CODE_SEARCH_DOMAINS",
"WebSearchError",
"SearchResult",
@@ -72,6 +85,12 @@ __all__ = [
DEFAULT_CHAIN = ("perplexity", "exa", "tavily", "brave", "searxng",
"duckduckgo")
# Paid backends, frozen out of the default auto chain (see the module
# docstring). DEFAULT_CHAIN keeps the legacy full order so the
# unfrozen chain is exactly the old one; DEFAULT_FREE_CHAIN is what
# default auto actually walks while frozen.
PAID_PROVIDERS = ("perplexity", "exa", "tavily", "brave")
DEFAULT_FREE_CHAIN = ("searxng", "duckduckgo")
DEFAULT_FETCH_CHAR_LIMIT = 15000
# Programming-only search scope (``search(..., scope="code")``).
@@ -246,6 +265,27 @@ class HttpClient:
raise WebSearchError(
"HTTP %s" % exc.code) from _ProviderHttpError(
exc.code, body)
except (socket.timeout, TimeoutError) as exc:
# A timeout is a timeout, never "no results": the caller
# (search's attempted entry, named by provider) must be
# able to tell which stage timed out, and a timed-out
# SearXNG must not be rewritten as an empty answer.
# HTTPError is a URLError subclass, so this branch must
# stay after the HTTPError branch above.
raise WebSearchError(
"timeout after %ss waiting for HTTP response"
% self.timeout) from exc
except urllib.error.URLError as exc:
reason = getattr(exc, "reason", None)
if isinstance(reason, (socket.timeout, TimeoutError)):
raise WebSearchError(
"timeout after %ss waiting for HTTP response"
% self.timeout) from exc
# DNS / connection failures keep their own message;
# they are not timeouts and must not be relabelled.
raise WebSearchError(
"URL error: %s" % reason if reason else "URL error"
) from exc
def get_text(self, url, headers=None):
status, body, _ctype = self.get(url, headers)
@@ -534,11 +574,32 @@ _PROVIDERS = {
}
def available_providers(env=None):
"""Providers usable right now: keyed ones with a key, plus free ones."""
def paid_search_enabled(env=None):
"""True only when the paid-unfreeze switch is explicitly on.
The switch is WEBSEARCH_ENABLE_PAID, and only the exact value
"1" counts (same strict convention as WEBSEARCH_ALLOW_PRIVATE_URLS
elsewhere in this module): any other value, including "true" or
"yes", leaves the paid backends frozen out of default auto.
Explicit pinning is unaffected by this switch either way.
"""
env = env if env is not None else os.environ
return env.get("WEBSEARCH_ENABLE_PAID") == "1"
def available_providers(env=None):
"""Providers usable right now, under the paid-freeze rule.
Frozen (default): only the free chain -- SearXNG when its URL
is set, plus DuckDuckGo -- even when paid keys are present.
Unfrozen (WEBSEARCH_ENABLE_PAID=1): the legacy full chain,
keyed backends only when their key is set, as before.
"""
env = env if env is not None else os.environ
chain = (DEFAULT_CHAIN if paid_search_enabled(env)
else DEFAULT_FREE_CHAIN)
out = []
for name in DEFAULT_CHAIN:
for name in chain:
if name in _FREE_PROVIDERS:
if name == "searxng" and not env.get("SEARXNG_URL"):
continue
@@ -552,8 +613,11 @@ def resolve_provider_chain(provider="auto", env=None):
"""Resolve the chain oh-my-pi style: auto walks, pinned does not.
``provider`` may be "auto" (default chain, filtered to what is
configured), a single name (that provider only, even if it fails),
or a comma-separated list (that order only). Unknown names raise
configured -- frozen by default to the free chain, SearXNG then
DuckDuckGo, unless WEBSEARCH_ENABLE_PAID=1 restores the legacy
full chain), a single name (that provider only, even if it fails
-- pinning a paid backend bypasses the freeze, deliberately), or
a comma-separated list (that order only). Unknown names raise
ValueError before any HTTP happens.
"""
env = env if env is not None else os.environ
+13 -3
View File
@@ -35,9 +35,19 @@ _TOOLS = [
"Search the web and return ranked results with title, URL "
"and snippet, plus a synthesised answer when the provider "
"supplies one. Use web_fetch to read specific pages. "
"provider may be 'auto' (walk Perplexity, Exa, Tavily, "
"Brave, then keyless SearXNG / DuckDuckGo, falling back "
"on failure) or a single provider name to pin it. "
"provider may be 'auto' (frozen default: keyless SearXNG "
"first, then DuckDuckGo as the free fallback, falling "
"back on failure; the paid backends Perplexity, Exa, "
"Tavily and Brave are NOT in the default chain even "
"when their keys are set, and a SearXNG failure never "
"falls back to a paid backend, so default use never "
"incurs paid cost) or a single provider name to pin it "
"(pinning a paid backend still calls it: that is an "
"explicit override, not a default). Set "
"WEBSEARCH_ENABLE_PAID=1 to restore the old default "
"chain (Perplexity, Exa, Tavily, Brave, SearXNG, "
"DuckDuckGo); unfreezing is explicit and may incur "
"paid cost. "
"scope 'code' restricts results to programming sources "
"only (official language docs, GitHub, package "
"registries, Stack Overflow, CVE/NVD); anything else "
+6
View File
@@ -3188,9 +3188,15 @@ class Worker:
# auto 链会跳过 SearXNG、只剩 DuckDuckGo(实弹 e2e 已证实)。
# 只透传已设置的项,值绝不进日志;付费 key 与网关 key 一样
# 走 extra_env(容器内进程本就需要它调 API)。
# WEBSEARCH_ENABLE_PAID rides along so the paid-freeze
# switch set on the host actually reaches the MCP process
# (which reads only config.toml env, never worker env).
# Default is frozen; the switch is honoured only when set
# to exactly "1" (see websearch.paid_search_enabled).
search_env = {
k: os.environ[k]
for k in ("SEARXNG_URL", "SEARXNG_ENGINES",
"WEBSEARCH_ENABLE_PAID",
"PERPLEXITY_API_KEY", "EXA_API_KEY",
"TAVILY_API_KEY", "BRAVE_API_KEY")
if os.environ.get(k)