websearch: 默认 auto 付费冻结,SearXNG 优先 + 显式解冻开关

- 默认 auto 只走免费链 searxng -> duckduckgo;即使环境里有
  付费 key 也不调用 Perplexity/Exa/Tavily/Brave,SearXNG
  失败也不回落付费,默认使用零付费花费。
- 付费后端代码/测试/文档全部保留;显式 pin(tavily/exa/
  perplexity/brave)仍照常调用,是有意覆盖不是默认。
- 解冻开关 WEBSEARCH_ENABLE_PAID,仅精确值 "1" 才恢复旧链
  (perplexity/exa/tavily/brave/searxng/duckduckgo),默认关;
  开关经 worker search_env 透传进 MCP config env。
- HttpClient 超时/DNS 失败改报显式错误(timeout after Ns /
  URL error),绝不把超时写成“无结果”。
- SearXNG 默认参数核实:代码无 SEARXNG_URL 硬编码默认值;
  SEARXNG_ENGINES 默认 bing,360search 已在基线;WEBSEARCH_TIMEOUT
  默认 20 秒。主机 env 实测仅 SEARXNG_URL=http://searx.diante.me
  一行,故默认 pin 生效;文档 https 漂移改回 http(主机 nginx
  仅听 80、无 443 vhost,https 在 TLS 层即失败)。
- 旧测试更新原因:test_websearch_mcp 原断言默认 auto 为
  perplexity 优先,与新冻结语义直接冲突,故改为断言冻结后
  的 searxng -> duckduckgo 回退 + 不泄漏 key;旧 perplexity ->
  exa 回退断言移至解冻开关测试(WEBSEARCH_ENABLE_PAID=1 下)。
  未删断言、未加 skip、未放宽门禁。
- 文档另删 WEBSEARCH_PROVIDERS 说法:代码从未实现该变量。
- 未推、未建 PR、未合并、未部署、未改主机 env/nginx/filter/
  证书、未做付费后端 live 验证、未创建生产任务。
This commit is contained in:
host committed 2026-11-01 14:15:54 +00:00
1 parent 0602d06136
commit 3a56f83114
7 files changed
+492 -43

No files matched your search

+27 -7
View File
@@ -67,18 +67,38 @@ def _exa_body():
class ChainTestCase(unittest.TestCase): class ChainTestCase(unittest.TestCase):
def test_fallback_perplexity_to_exa_and_key_not_leaked(self): def test_default_auto_frozen_falls_back_searxng_to_duckduckgo_and_key_not_leaked(self):
# Paid-freeze update of the old perplexity -> exa fallback
# test: default auto no longer contains any paid backend,
# even when all four paid keys are set, so the fallback
# that still exists by default is searxng -> duckduckgo
# and no paid URL is ever touched. The old perplexity ->
# exa fallback is preserved, but only under
# WEBSEARCH_ENABLE_PAID=1; see test_websearch_paid_freeze.py.
html = (
'<a class="result__a" href="//duckduckgo.com/l/?uddg='
'https%3A%2F%2Fexample.com%2Fddg">DDG Title</a>'
'<a class="result__snippet">ddg snippet</a>')
http = FakeHttp([ http = FakeHttp([
("perplexity", {}, 500), ("searx", {}, 500),
("api.exa.ai", _exa_body(), 200), ("duckduckgo", html, 200),
]) ])
env = dict(_FAKE_KEYS)
env["SEARXNG_URL"] = "http://searx.test"
out = search("test query", provider="auto", out = search("test query", provider="auto",
env=dict(_FAKE_KEYS), http=http) env=env, http=http)
self.assertEqual(out["provider"], "exa") self.assertEqual(out["provider"], "duckduckgo")
self.assertEqual(out["results"][0]["url"], self.assertEqual(out["results"][0]["url"],
"https://example.com/exa") "https://example.com/ddg")
self.assertEqual(out["attempted"][0]["provider"], "perplexity") self.assertEqual(out["attempted"][0]["provider"], "searxng")
self.assertFalse(out["attempted"][0]["ok"]) self.assertFalse(out["attempted"][0]["ok"])
self.assertEqual(
[a["provider"] for a in out["attempted"]],
["searxng", "duckduckgo"])
for _method, url, _payload in http.seen:
for paid in ("perplexity", "exa.ai", "tavily",
"search.brave.com"):
self.assertNotIn(paid, url)
blob = json.dumps(out) blob = json.dumps(out)
for secret in _FAKE_KEYS.values(): for secret in _FAKE_KEYS.values():
self.assertNotIn(secret, blob) self.assertNotIn(secret, blob)
+280
View File
@@ -0,0 +1,280 @@
"""Paid-freeze tests: default auto is free-only, pin still works,
unfreeze is explicit (WEBSEARCH_ENABLE_PAID=1, exact value "1").
No network, no real keys: every HTTP call goes through a FakeHttp,
and the fake key values are deliberately distinctive so a leak into
an error string or a tool result would be caught by assertion.
"""
import json
import os
import socket
import sys
import unittest
import urllib.error
from pathlib import Path
from unittest import mock
try:
from cloud.websearch import (
DEFAULT_CHAIN, DEFAULT_FREE_CHAIN, HttpClient, WebSearchError,
available_providers, paid_search_enabled,
resolve_provider_chain, search)
from cloud.websearch_mcp import handle_request
except ImportError:
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
from cloud.websearch import (
DEFAULT_CHAIN, DEFAULT_FREE_CHAIN, HttpClient, WebSearchError,
available_providers, paid_search_enabled,
resolve_provider_chain, search)
from cloud.websearch_mcp import handle_request
_FAKE_KEYS = {
"PERPLEXITY_API_KEY": "pplx-SECRET-111",
"EXA_API_KEY": "exa-SECRET-222",
"TAVILY_API_KEY": "tvly-SECRET-333",
"BRAVE_API_KEY": "brave-SECRET-444",
}
_DDG_HTML = (
'<a class="result__a" href="//duckduckgo.com/l/?uddg='
'https%3A%2F%2Fexample.com%2Fddg">DDG Title</a>'
'<a class="result__snippet">ddg snippet</a>')
class FakeHttp:
"""post_json/get keyed by URL substring; failures raise like HTTP."""
def __init__(self, routes):
self.routes = routes
self.seen = []
def post_json(self, url, headers, payload):
self.seen.append(("POST", url, payload))
return self._route(url)
def get(self, url, headers=None):
self.seen.append(("GET", url, headers))
return self._route(url)
def _route(self, url):
for needle, body, status in self.routes:
if needle in url:
if status != 200:
raise WebSearchError("HTTP %s" % status)
if isinstance(body, dict):
return 200, json.dumps(body).encode(), "application/json"
return 200, body.encode(), "text/html"
raise WebSearchError("HTTP 404")
def _searxng_body():
return {"results": [
{"title": "SearXNG Hit", "url": "https://example.com/sx",
"content": "searxng snippet"}]}
def _paid_env(**extra):
env = dict(_FAKE_KEYS)
env["SEARXNG_URL"] = "http://searx.test"
env.update(extra)
return env
def _paid_hosts_seen(http):
hosts = []
for _method, url, _payload in http.seen:
for paid in ("perplexity", "exa.ai", "tavily",
"search.brave.com"):
if paid in url:
hosts.append(paid)
return hosts
class FrozenDefaultTestCase(unittest.TestCase):
def test_default_auto_excludes_paid_even_with_all_keys(self):
env = _paid_env()
self.assertEqual(available_providers(env),
["searxng", "duckduckgo"])
self.assertEqual(resolve_provider_chain("auto", env),
["searxng", "duckduckgo"])
http = FakeHttp([("searx", _searxng_body(), 200)])
out = search("q", provider="auto", env=env, http=http)
self.assertEqual(out["provider"], "searxng")
self.assertEqual(_paid_hosts_seen(http), [])
def test_default_auto_without_searxng_url_is_duckduckgo_only(self):
env = dict(_FAKE_KEYS)
self.assertEqual(resolve_provider_chain("auto", env),
["duckduckgo"])
http = FakeHttp([("duckduckgo", _DDG_HTML, 200)])
out = search("q", provider="auto", env=env, http=http)
self.assertEqual(out["provider"], "duckduckgo")
self.assertEqual(_paid_hosts_seen(http), [])
def test_searxng_failure_never_falls_back_to_paid(self):
http = FakeHttp([("searx", {}, 500),
("duckduckgo", "<html>challenge</html>", 200)])
with self.assertRaises(WebSearchError) as ctx:
search("q", provider="auto", env=_paid_env(), http=http)
self.assertEqual(
[a["provider"] for a in ctx.exception.attempted],
["searxng", "duckduckgo"])
self.assertEqual(_paid_hosts_seen(http), [])
# No fabricated result: the error names both free stages.
self.assertIn("searxng", str(ctx.exception))
self.assertIn("duckduckgo", str(ctx.exception))
def test_duckduckgo_fallback_failure_reports_clearly(self):
# DuckDuckGo under the VM/host egress returns a 202-style
# robot-challenge page: the parser yields no results, which
# must surface as an honest "no results" failure for that
# stage, never as a fabricated hit.
http = FakeHttp([("searx", {}, 500),
("duckduckgo", "<html>anomaly</html>", 200)])
with self.assertRaises(WebSearchError) as ctx:
search("q", provider="auto", env=_paid_env(), http=http)
attempted = {a["provider"]: a for a in ctx.exception.attempted}
self.assertEqual(attempted["duckduckgo"]["error"], "no results")
self.assertFalse(attempted["duckduckgo"]["ok"])
for secret in _FAKE_KEYS.values():
self.assertNotIn(secret, str(ctx.exception))
class ExplicitPinTestCase(unittest.TestCase):
def test_explicit_pin_paid_still_works_while_frozen(self):
http = FakeHttp([
("api.tavily.com",
{"results": [{"title": "T", "url": "https://example.com/t",
"content": "tavily content"}]}, 200),
])
out = search("q", provider="tavily", env=_paid_env(), http=http)
self.assertEqual(out["provider"], "tavily")
self.assertEqual(out["results"][0]["snippet"], "tavily content")
def test_explicit_pin_paid_failure_does_not_fall_back(self):
http = FakeHttp([("perplexity", {}, 500),
("api.exa.ai", {"results": [
{"title": "E", "url": "https://example.com/e",
"text": "x"}]}, 200)])
with self.assertRaises(WebSearchError):
search("q", provider="perplexity", env=_paid_env(), http=http)
self.assertEqual(_paid_hosts_seen(http), ["perplexity"])
def test_frozen_default_applies_to_scope_code_too(self):
body = {"results": [
{"title": "GH", "url": "https://github.com/a/b",
"content": "code hit"},
{"title": "Other", "url": "https://example.com/other",
"content": "off-scope"}]}
http = FakeHttp([("searx", body, 200)])
out = search("q", provider="auto", env=_paid_env(),
http=http, scope="code")
self.assertEqual(out["provider"], "searxng")
self.assertEqual(
[r["url"] for r in out["results"]], ["https://github.com/a/b"])
self.assertEqual(_paid_hosts_seen(http), [])
class UnfreezeSwitchTestCase(unittest.TestCase):
def test_switch_off_by_default_and_non_1_values_do_not_unfreeze(self):
self.assertFalse(paid_search_enabled({}))
for value in ("0", "true", "yes", "", "on"):
env = _paid_env(WEBSEARCH_ENABLE_PAID=value)
self.assertFalse(paid_search_enabled(env), value)
self.assertEqual(resolve_provider_chain("auto", env),
["searxng", "duckduckgo"])
def test_switch_on_restores_legacy_chain_and_fallback(self):
env = _paid_env(WEBSEARCH_ENABLE_PAID="1")
self.assertTrue(paid_search_enabled(env))
self.assertEqual(resolve_provider_chain("auto", env),
list(DEFAULT_CHAIN))
# The pre-freeze perplexity -> exa fallback, preserved
# verbatim under the explicit switch (plus the no-leak
# assertion from the old default-chain test).
http = FakeHttp([
("perplexity", {}, 500),
("api.exa.ai", {"results": [
{"title": "Exa Hit", "url": "https://example.com/exa",
"text": "exa snippet", "score": 0.9}]}, 200),
])
out = search("test query", provider="auto", env=env, http=http)
self.assertEqual(out["provider"], "exa")
self.assertEqual(out["results"][0]["url"],
"https://example.com/exa")
self.assertEqual(out["attempted"][0]["provider"], "perplexity")
self.assertFalse(out["attempted"][0]["ok"])
blob = json.dumps(out)
for secret in _FAKE_KEYS.values():
self.assertNotIn(secret, blob)
def test_constants_state_the_contract(self):
self.assertEqual(DEFAULT_FREE_CHAIN, ("searxng", "duckduckgo"))
self.assertEqual(
DEFAULT_CHAIN,
("perplexity", "exa", "tavily", "brave", "searxng",
"duckduckgo"))
class TimeoutClarityTestCase(unittest.TestCase):
def test_http_client_maps_timeout_to_explicit_error(self):
client = HttpClient(timeout=20)
with mock.patch("urllib.request.urlopen",
side_effect=socket.timeout("timed out")):
with self.assertRaises(WebSearchError) as ctx:
client.get("http://searx.test/search?q=x")
self.assertIn("timeout after 20s", str(ctx.exception))
with mock.patch(
"urllib.request.urlopen",
side_effect=urllib.error.URLError(
socket.timeout("timed out"))):
with self.assertRaises(WebSearchError) as ctx:
client.get("http://searx.test/search?q=x")
self.assertIn("timeout after 20s", str(ctx.exception))
def test_http_client_does_not_relabel_dns_failure_as_timeout(self):
client = HttpClient(timeout=20)
with mock.patch(
"urllib.request.urlopen",
side_effect=urllib.error.URLError(
"Temporary failure in name resolution")):
with self.assertRaises(WebSearchError) as ctx:
client.get("http://searx.test/search?q=x")
self.assertNotIn("timeout", str(ctx.exception))
self.assertIn("name resolution", str(ctx.exception))
def test_search_reports_timeout_as_error_not_no_results(self):
class TimeoutHttp(FakeHttp):
def get(self, url, headers=None):
self.seen.append(("GET", url, headers))
if "searx" in url:
raise WebSearchError(
"timeout after 20s waiting for HTTP response")
return self._route(url)
http = TimeoutHttp([("duckduckgo", _DDG_HTML, 200)])
out = search("q", provider="auto", env=_paid_env(), http=http)
self.assertEqual(out["provider"], "duckduckgo")
attempted = {a["provider"]: a for a in out["attempted"]}
self.assertIn("timeout after 20s",
attempted["searxng"]["error"])
self.assertNotEqual(attempted["searxng"]["error"], "no results")
class McpDescriptionTestCase(unittest.TestCase):
def test_tools_list_describes_frozen_default_and_unfreeze(self):
resp = handle_request(
{"jsonrpc": "2.0", "id": 1, "method": "tools/list"})
tool = [t for t in resp["result"]["tools"]
if t["name"] == "web_search"][0]
desc = tool["description"]
self.assertIn("SearXNG", desc)
self.assertIn("DuckDuckGo", desc)
self.assertIn("WEBSEARCH_ENABLE_PAID=1", desc)
self.assertIn("paid", desc.lower())
if __name__ == "__main__":
unittest.main()
+15 -1
View File
@@ -124,10 +124,24 @@ class DockerEnvTestCase(unittest.TestCase):
self.assertIsNone(err) self.assertIsNone(err)
self.assertEqual(env.get("SEARXNG_URL"), "http://searx.diante.me") self.assertEqual(env.get("SEARXNG_URL"), "http://searx.diante.me")
def test_paid_unfreeze_switch_passes_into_container_env(self):
# The freeze switch must reach the MCP process (which reads
# only config.toml env, never worker env); without this
# passthrough, WEBSEARCH_ENABLE_PAID=1 on the host would
# silently leave default auto frozen inside containers.
from unittest import mock
with mock.patch.dict(os.environ,
{"WEBSEARCH_ENABLE_PAID": "1"}):
ok, _, _, env, err = self._docker_context()
self.assertTrue(ok)
self.assertIsNone(err)
self.assertEqual(env.get("WEBSEARCH_ENABLE_PAID"), "1")
def test_no_search_env_means_no_extra_env(self): def test_no_search_env_means_no_extra_env(self):
from unittest import mock from unittest import mock
with mock.patch.dict(os.environ, {}, clear=False): with mock.patch.dict(os.environ, {}, clear=False):
for k in ("SEARXNG_URL", "PERPLEXITY_API_KEY", "EXA_API_KEY", for k in ("SEARXNG_URL", "WEBSEARCH_ENABLE_PAID",
"PERPLEXITY_API_KEY", "EXA_API_KEY",
"TAVILY_API_KEY", "BRAVE_API_KEY"): "TAVILY_API_KEY", "BRAVE_API_KEY"):
os.environ.pop(k, None) os.environ.pop(k, None)
ok, _, _, env, err = self._docker_context() ok, _, _, env, err = self._docker_context()
+69 -5
View File
@@ -21,6 +21,16 @@ Design notes (what is copied from oh-my-pi, and what is not):
here is a small stdlib HTML parser: script/style stripped, text here is a small stdlib HTML parser: script/style stripped, text
collapsed, long pages truncated head+tail like Hermes' deterministic collapsed, long pages truncated head+tail like Hermes' deterministic
budget (default 15000 chars, 75% head / 25% tail, ``[TRUNCATED]``). budget (default 15000 chars, 75% head / 25% tail, ``[TRUNCATED]``).
- FROZEN default chain (paid freeze): the default ``auto`` chain
contains only the free backends, SearXNG then DuckDuckGo, even
when paid keys happen to be set -- a SearXNG failure never falls
back to a paid backend, so default use can never incur paid cost.
The four paid backends (Perplexity, Exa, Tavily, Brave) keep their
code, tests and docs, and still run when explicitly pinned
(provider="tavily" etc.), which is a deliberate override, not a
default. Only WEBSEARCH_ENABLE_PAID=1 (exact value "1", default
off) restores the old default chain; unfreezing is an explicit act
and may incur paid cost, never a silent default.
Security notes: Security notes:
- API keys come only from the process environment, never from - API keys come only from the process environment, never from
@@ -57,6 +67,9 @@ from html.parser import HTMLParser
__all__ = [ __all__ = [
"DEFAULT_CHAIN", "DEFAULT_CHAIN",
"DEFAULT_FREE_CHAIN",
"PAID_PROVIDERS",
"paid_search_enabled",
"CODE_SEARCH_DOMAINS", "CODE_SEARCH_DOMAINS",
"WebSearchError", "WebSearchError",
"SearchResult", "SearchResult",
@@ -72,6 +85,12 @@ __all__ = [
DEFAULT_CHAIN = ("perplexity", "exa", "tavily", "brave", "searxng", DEFAULT_CHAIN = ("perplexity", "exa", "tavily", "brave", "searxng",
"duckduckgo") "duckduckgo")
# Paid backends, frozen out of the default auto chain (see the module
# docstring). DEFAULT_CHAIN keeps the legacy full order so the
# unfrozen chain is exactly the old one; DEFAULT_FREE_CHAIN is what
# default auto actually walks while frozen.
PAID_PROVIDERS = ("perplexity", "exa", "tavily", "brave")
DEFAULT_FREE_CHAIN = ("searxng", "duckduckgo")
DEFAULT_FETCH_CHAR_LIMIT = 15000 DEFAULT_FETCH_CHAR_LIMIT = 15000
# Programming-only search scope (``search(..., scope="code")``). # Programming-only search scope (``search(..., scope="code")``).
@@ -246,6 +265,27 @@ class HttpClient:
raise WebSearchError( raise WebSearchError(
"HTTP %s" % exc.code) from _ProviderHttpError( "HTTP %s" % exc.code) from _ProviderHttpError(
exc.code, body) exc.code, body)
except (socket.timeout, TimeoutError) as exc:
# A timeout is a timeout, never "no results": the caller
# (search's attempted entry, named by provider) must be
# able to tell which stage timed out, and a timed-out
# SearXNG must not be rewritten as an empty answer.
# HTTPError is a URLError subclass, so this branch must
# stay after the HTTPError branch above.
raise WebSearchError(
"timeout after %ss waiting for HTTP response"
% self.timeout) from exc
except urllib.error.URLError as exc:
reason = getattr(exc, "reason", None)
if isinstance(reason, (socket.timeout, TimeoutError)):
raise WebSearchError(
"timeout after %ss waiting for HTTP response"
% self.timeout) from exc
# DNS / connection failures keep their own message;
# they are not timeouts and must not be relabelled.
raise WebSearchError(
"URL error: %s" % reason if reason else "URL error"
) from exc
def get_text(self, url, headers=None): def get_text(self, url, headers=None):
status, body, _ctype = self.get(url, headers) status, body, _ctype = self.get(url, headers)
@@ -534,11 +574,32 @@ _PROVIDERS = {
} }
def available_providers(env=None): def paid_search_enabled(env=None):
"""Providers usable right now: keyed ones with a key, plus free ones.""" """True only when the paid-unfreeze switch is explicitly on.
The switch is WEBSEARCH_ENABLE_PAID, and only the exact value
"1" counts (same strict convention as WEBSEARCH_ALLOW_PRIVATE_URLS
elsewhere in this module): any other value, including "true" or
"yes", leaves the paid backends frozen out of default auto.
Explicit pinning is unaffected by this switch either way.
"""
env = env if env is not None else os.environ env = env if env is not None else os.environ
return env.get("WEBSEARCH_ENABLE_PAID") == "1"
def available_providers(env=None):
"""Providers usable right now, under the paid-freeze rule.
Frozen (default): only the free chain -- SearXNG when its URL
is set, plus DuckDuckGo -- even when paid keys are present.
Unfrozen (WEBSEARCH_ENABLE_PAID=1): the legacy full chain,
keyed backends only when their key is set, as before.
"""
env = env if env is not None else os.environ
chain = (DEFAULT_CHAIN if paid_search_enabled(env)
else DEFAULT_FREE_CHAIN)
out = [] out = []
for name in DEFAULT_CHAIN: for name in chain:
if name in _FREE_PROVIDERS: if name in _FREE_PROVIDERS:
if name == "searxng" and not env.get("SEARXNG_URL"): if name == "searxng" and not env.get("SEARXNG_URL"):
continue continue
@@ -552,8 +613,11 @@ def resolve_provider_chain(provider="auto", env=None):
"""Resolve the chain oh-my-pi style: auto walks, pinned does not. """Resolve the chain oh-my-pi style: auto walks, pinned does not.
``provider`` may be "auto" (default chain, filtered to what is ``provider`` may be "auto" (default chain, filtered to what is
configured), a single name (that provider only, even if it fails), configured -- frozen by default to the free chain, SearXNG then
or a comma-separated list (that order only). Unknown names raise DuckDuckGo, unless WEBSEARCH_ENABLE_PAID=1 restores the legacy
full chain), a single name (that provider only, even if it fails
-- pinning a paid backend bypasses the freeze, deliberately), or
a comma-separated list (that order only). Unknown names raise
ValueError before any HTTP happens. ValueError before any HTTP happens.
""" """
env = env if env is not None else os.environ env = env if env is not None else os.environ
+13 -3
View File
@@ -35,9 +35,19 @@ _TOOLS = [
"Search the web and return ranked results with title, URL " "Search the web and return ranked results with title, URL "
"and snippet, plus a synthesised answer when the provider " "and snippet, plus a synthesised answer when the provider "
"supplies one. Use web_fetch to read specific pages. " "supplies one. Use web_fetch to read specific pages. "
"provider may be 'auto' (walk Perplexity, Exa, Tavily, " "provider may be 'auto' (frozen default: keyless SearXNG "
"Brave, then keyless SearXNG / DuckDuckGo, falling back " "first, then DuckDuckGo as the free fallback, falling "
"on failure) or a single provider name to pin it. " "back on failure; the paid backends Perplexity, Exa, "
"Tavily and Brave are NOT in the default chain even "
"when their keys are set, and a SearXNG failure never "
"falls back to a paid backend, so default use never "
"incurs paid cost) or a single provider name to pin it "
"(pinning a paid backend still calls it: that is an "
"explicit override, not a default). Set "
"WEBSEARCH_ENABLE_PAID=1 to restore the old default "
"chain (Perplexity, Exa, Tavily, Brave, SearXNG, "
"DuckDuckGo); unfreezing is explicit and may incur "
"paid cost. "
"scope 'code' restricts results to programming sources " "scope 'code' restricts results to programming sources "
"only (official language docs, GitHub, package " "only (official language docs, GitHub, package "
"registries, Stack Overflow, CVE/NVD); anything else " "registries, Stack Overflow, CVE/NVD); anything else "
+6
View File
@@ -3188,9 +3188,15 @@ class Worker:
# auto 链会跳过 SearXNG、只剩 DuckDuckGo(实弹 e2e 已证实)。 # auto 链会跳过 SearXNG、只剩 DuckDuckGo(实弹 e2e 已证实)。
# 只透传已设置的项,值绝不进日志;付费 key 与网关 key 一样 # 只透传已设置的项,值绝不进日志;付费 key 与网关 key 一样
# 走 extra_env(容器内进程本就需要它调 API)。 # 走 extra_env(容器内进程本就需要它调 API)。
# WEBSEARCH_ENABLE_PAID rides along so the paid-freeze
# switch set on the host actually reaches the MCP process
# (which reads only config.toml env, never worker env).
# Default is frozen; the switch is honoured only when set
# to exactly "1" (see websearch.paid_search_enabled).
search_env = { search_env = {
k: os.environ[k] k: os.environ[k]
for k in ("SEARXNG_URL", "SEARXNG_ENGINES", for k in ("SEARXNG_URL", "SEARXNG_ENGINES",
"WEBSEARCH_ENABLE_PAID",
"PERPLEXITY_API_KEY", "EXA_API_KEY", "PERPLEXITY_API_KEY", "EXA_API_KEY",
"TAVILY_API_KEY", "BRAVE_API_KEY") "TAVILY_API_KEY", "BRAVE_API_KEY")
if os.environ.get(k) if os.environ.get(k)
+82 -27
View File
@@ -20,6 +20,11 @@ pi 的工具),只复用它的模式:统一工具 + provider 抽象 +
- `cloud/tests/test_code_search_ext.py`:任务级 - `cloud/tests/test_code_search_ext.py`:任务级
`allowed_hosts` 扩展的单测(指定 / 未指定 / 点边界 / `allowed_hosts` 扩展的单测(指定 / 未指定 / 点边界 /
空值与非法值 / Perplexity 超 20 域 / MCP 透传)。 空值与非法值 / Perplexity 超 20 域 / MCP 透传)。
- `cloud/tests/test_websearch_paid_freeze.py`:付费冻结单测
(默认 auto 无付费后端、SearXNG 失败不回落付费、显式
pin 仍工作、解冻开关关时不变 / 开时恢复旧链、
DuckDuckGo 兜底失败报错清晰、超时报错清晰;全部
mock HTTP,不联网、不需要真实 key)。
- `cloud/tests/test_js_heavy_extract.py`:JS 重型页面 - `cloud/tests/test_js_heavy_extract.py`:JS 重型页面
抽取评估的 mock 单测(直接抓取壳-only、provider 抽取 抽取评估的 mock 单测(直接抓取壳-only、provider 抽取
补与不补、回退语义;live SearXNG 烟囱默认跳过)。 补与不补、回退语义;live SearXNG 烟囱默认跳过)。
@@ -39,7 +44,48 @@ pi 的工具),只复用它的模式:统一工具 + provider 抽象 +
超长按头 75% + 尾 25% 截断并标 `[TRUNCATED]`;只接受 http(s),拒绝 超长按头 75% + 尾 25% 截断并标 `[TRUNCATED]`;只接受 http(s),拒绝
localhost 与字面私有 IP;抽取与直接抓取都失败时报错,不伪造页面。 localhost 与字面私有 IP;抽取与直接抓取都失败时报错,不伪造页面。
## Provider 链(auto 顺序) ## Provider 链:默认冻结,显式 pin 与显式解冻除外
冻结语义(本轨起,默认 `auto` 只走免费路线,顺序固定):
1. `searxng`:`GET $SEARXNG_URL/search?format=json`,需自托管
实例且 JSON 格式已打开。默认参数:`engines=bing,360search`
(2026-10-30 主机实测:默认 general 类别 20.0s 撞超时,
pinned 两引擎 0.47s;`SEARXNG_ENGINES` 可覆盖,空值恢复
不 pin)。`SEARXNG_URL` 在代码里无默认值:未配时该后端
直接报 `searxng not configured (SEARXNG_URL unset)`,链
只剩 DuckDuckGo;生产值 `http://searx.diante.me` 来自主
机 env(见“生产接线”),不在代码里写死;主机 env 实测
仅有 `SEARXNG_URL` 这一行、无 `SEARXNG_ENGINES` 行,
所以默认 pin(bing,360search)生效。注意是 http 不是
https:主机 nginx 只听 80、无 443 vhost,https 会在 TLS
层证书主机名不匹配失败,根本到不了 SearXNG。
2. `duckduckgo`:解析 `https://html.duckduckgo.com/html/`,
免 key 兜底,布局漂移时只会返回空结果并继续走链,
不会把工具打崩;但它在两种出口下都不可作可用兜底:
本 VM 直连回 202 + 机器人挑战页、0 条,主机 Tinyproxy
路径约 25 秒超时。兜底失败时记为 `no results` / 超时并
如实报错,不伪造结果;因此 SearXNG 挂时冻结默认链可能
以诚实失败告终,这是已知边界,不是静默降级。
冻结保证:即使环境里配了付费 key,默认 auto 也绝不调用
Perplexity / Exa / Tavily / Brave、绝不产生付费花费;
SearXNG 失败后默认也不回落到付费后端,只走 DuckDuckGo。
付费后端的代码、测试、文档一律保留,未删除。
显式 pin 仍工作:`provider="tavily"` / `"exa"` /
`"perplexity"` / `"brave"` 显式指定时照常调用,这是用户
有意的覆盖,不是默认行为;pin 失败不回退(与冻结前一致)。
解冻开关:`WEBSEARCH_ENABLE_PAID=1`(且仅精确值 `1`,
`true` / `yes` / `0` 都不算开,默认关)时,默认 auto
恢复旧链,顺序为 Perplexity → Exa → Tavily → Brave →
SearXNG → DuckDuckGo(旧链里只试有 key 的付费后端 +
已配 URL 的 SearXNG + DuckDuckGo)。解冻是显式动作,
会产生付费花费,绝不能被默认链静默触发;开关走 worker
环境透传进 MCP config env(见“生产接线”)。
旧链各后端(解冻后 / 显式 pin 时)的请求形态不变:
1. `perplexity`:`POST https://api.perplexity.ai/chat/completions`, 1. `perplexity`:`POST https://api.perplexity.ai/chat/completions`,
模型 `sonar`,`Authorization: Bearer $PERPLEXITY_API_KEY`。 模型 `sonar`,`Authorization: Bearer $PERPLEXITY_API_KEY`。
@@ -53,18 +99,16 @@ pi 的工具),只复用它的模式:统一工具 + provider 抽象 +
结果,总结留给模型)。 结果,总结留给模型)。
4. `brave`:`GET https://api.search.brave.com/res/v1/web/search`, 4. `brave`:`GET https://api.search.brave.com/res/v1/web/search`,
头 `X-Subscription-Token`,`extra_snippets: true`。 头 `X-Subscription-Token`,`extra_snippets: true`。
5. `searxng`:`GET $SEARXNG_URL/search?format=json`,需自托管
实例且 JSON 格式已打开。
6. `duckduckgo`:解析 `https://html.duckduckgo.com/html/`,
免 key 兜底,布局漂移时只会返回空结果并继续走链,
不会把工具打崩。
规则:auto 只试已配置的后端(有 key 的付费后端 + 已配 规则:任一后端 HTTP 失败或 0 结果都继续下一个;全部失败
URL 的 SearXNG + DuckDuckGo);任一后端 HTTP 失败或 返回各后端错误(不含 key)。超时失败报
0 结果都继续下一个;全部失败返回各后端错误(不含 key)。 `timeout after Ns waiting for HTTP response`(N 为
`provider` pin 单个名字时不回退,失败即报错。 `WEBSEARCH_TIMEOUT`,默认 20 秒),并记在该后端的
可用 `WEBSEARCH_PROVIDERS`(逗号列表)改默认链顺序, attempted 条目里,绝不把超时写成“无结果”或空答案。
`WEBSEARCH_TIMEOUT`(秒,默认 20)改单次超时。 `WEBSEARCH_TIMEOUT`(秒,默认 20)改单次超时。另注意:
旧文档曾写可用 `WEBSEARCH_PROVIDERS` 改默认链顺序,
代码里从未实现该变量,本轨已删此说法(不另新加该变量,
保持改动最小)。
## 接入 codex(测试项目做法) ## 接入 codex(测试项目做法)
@@ -97,15 +141,19 @@ Docker 租户必须传容器内路径
- egress 白名单:`deploy/egress/tinyproxy-filter` 已加 - egress 白名单:`deploy/egress/tinyproxy-filter` 已加
Perplexity、Exa、Tavily、Brave、DuckDuckGo 与 Perplexity、Exa、Tavily、Brave、DuckDuckGo 与
`searx.diante.me`(精确匹配,不用 diante.me 通配,否则 `searx.diante.me`(精确匹配,不用 diante.me 通配,否则
任务经代理也能摸到 api.diante.me 控制面)。主机上须把 任务经代理也能摸到 api.diante.me 控制面;filter 行即
`^searx\.diante\.me$`,本轨不改这行)。主机上须把
该文件拷到 `/etc/tinyproxy/tinyproxy-filter` 并重载 该文件拷到 `/etc/tinyproxy/tinyproxy-filter` 并重载
tinyproxy,重载后先验中转域名还通,再验新域名。 tinyproxy,重载后先验中转域名还通,再验新域名。
- 自托管 SearXNG(公开子域名路):主机 Docker 跑 - 自托管 SearXNG(公开子域名路):主机 Docker 跑
`searxng/searxng`,经 nginx 反代到 `searx.diante.me` `searxng/searxng`,经 nginx(仅听 80,无 443 vhost、无
(DNS A 记录 + certbot 证书),MCP 侧 证书)反代到 `searx.diante.me`,MCP 侧
`SEARXNG_URL=https://searx.diante.me` 走 worker 进程 `SEARXNG_URL=http://searx.diante.me` 走 worker 进程
环境(`/etc/codex-app-server/env`)继承到 codex 再到 环境(`/etc/codex-app-server/env`)继承到 codex 再到
MCP,不写进 config.toml。公开实例的代价是可能被外人 MCP,不写进 config.toml。`WEBSEARCH_ENABLE_PAID` 同理
走 worker 的 search_env 透传进 `[mcp_servers.websearch.env]`
(worker 代码已含该键);主机侧改 env 文件后需重启相关
unit 才生效——那是部署动作,不在本轨。公开实例的代价是可能被外人
刷:SearXNG limiter 保持开启,JSON 格式是 MCP 要用的 刷:SearXNG limiter 保持开启,JSON 格式是 MCP 要用的
唯一格式。未加域名进白名单之前,Docker 任务的搜索 唯一格式。未加域名进白名单之前,Docker 任务的搜索
会被代理 403 挡住(fail closed,工具报错,不会静默绕过)。 会被代理 403 挡住(fail closed,工具报错,不会静默绕过)。
@@ -267,6 +315,10 @@ filter 行须用 POSIX BASIC 正则(`^\(.*\.\)*域名$`)、
- 本批只做代码 + 单测 + 测试租户目录的配置合并验证, - 本批只做代码 + 单测 + 测试租户目录的配置合并验证,
未推分支、未建 PR、未合并、未部署、未改生产 worker 未推分支、未建 PR、未合并、未部署、未改生产 worker
与 egress 配置、未跑真实 codex 任务。 与 egress 配置、未跑真实 codex 任务。
- 付费冻结本轨同样只在本地提交里:生产默认链要等部署
新代码后才变;解冻开关在主机上默认仍是关(除非主机
env 显式设为 `WEBSEARCH_ENABLE_PAID=1`,且那需要
主机侧改 env + 重启 unit,属部署动作)。
- DuckDuckGo 的 HTML 解析是兜底手段,布局一变就可能 - DuckDuckGo 的 HTML 解析是兜底手段,布局一变就可能
0 结果;它存在的意义是“没 key 也能走通链路”,不是 0 结果;它存在的意义是“没 key 也能走通链路”,不是
长期依赖的搜索后端。实测(2026-10-28,VM 出口): 长期依赖的搜索后端。实测(2026-10-28,VM 出口):
@@ -335,16 +387,19 @@ mock 补不了、也证明不了的部分:
### Live 对照(与 mock 分开写,未验证的不写成已验证) ### Live 对照(与 mock 分开写,未验证的不写成已验证)
- 无 key 的 SearXNG 路线:本轨在 VM 测试环境实测一次 - 无 key 的 SearXNG 路线:旧 VM 测试曾用
(`SEARXNG_URL=https://searx.diante.me`,`provider="searxng"`, `SEARXNG_URL=https://searx.diante.me` 失败(`URLError:
产品自身 `search()` 代码):失败,`URLError: Remote end Remote end closed connection without response`)——该失败
closed connection without response`。这是在本 VM 出口下 现已归因:https 是错的,主机 nginx 只听 80、无 443
的一次结果,不能据此断言 SearXNG 服务本身挂了(生产 vhost,https 在 TLS 层证书主机名不匹配失败,根本没到
文档记载主机侧经代理 pinned 引擎曾 0.47s 通),live SearXNG,不能据此断言服务挂了。2026-11-01 探针实测:
对照在本测试环境记为**未完成**,需在主机侧可达的环境 http 经同一 Tinyproxy 0.34 秒回 200、16 条结果,服务
重测;测试文件里的 live 烟囱测试默认跳过,只在 活着;主机 env 仅 `SEARXNG_URL=http://searx.diante.me`
`WEBSEARCH_EVAL_LIVE_SEARXNG=1` 且配了 `SEARXNG_URL` 一行、无 `SEARXNG_ENGINES`,默认 pin 生效。容器内真实
时才跑。 任务派发仍是 blocker(不由本轨创建生产任务),live
对照在该项上仍记为**未完成**;测试文件里的 live 烟囱
测试默认跳过,只在 `WEBSEARCH_EVAL_LIVE_SEARXNG=1`
且配了 `SEARXNG_URL` 时才跑。
- 付费后端 live 效果(Tavily extract / Exa contents / - 付费后端 live 效果(Tavily extract / Exa contents /
Perplexity / Brave 的真实 JS 渲染能力):无 key,本轨 Perplexity / Brave 的真实 JS 渲染能力):无 key,本轨
**一律未验证**,不编造效果,逐项列为 blocker(见回执), **一律未验证**,不编造效果,逐项列为 blocker(见回执),