From 3a56f831141db98a35d80815f726e82239bb2900 Mon Sep 17 00:00:00 2001 From: host Date: Sun, 1 Nov 2026 14:15:54 +0000 Subject: [PATCH] =?UTF-8?q?websearch:=20=E9=BB=98=E8=AE=A4=20auto=20?= =?UTF-8?q?=E4=BB=98=E8=B4=B9=E5=86=BB=E7=BB=93=EF=BC=8CSearXNG=20?= =?UTF-8?q?=E4=BC=98=E5=85=88=20+=20=E6=98=BE=E5=BC=8F=E8=A7=A3=E5=86=BB?= =?UTF-8?q?=E5=BC=80=E5=85=B3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 默认 auto 只走免费链 searxng -> duckduckgo；即使环境里有 付费 key 也不调用 Perplexity/Exa/Tavily/Brave，SearXNG 失败也不回落付费，默认使用零付费花费。 - 付费后端代码/测试/文档全部保留；显式 pin（tavily/exa/ perplexity/brave）仍照常调用，是有意覆盖不是默认。 - 解冻开关 WEBSEARCH_ENABLE_PAID，仅精确值 "1" 才恢复旧链 （perplexity/exa/tavily/brave/searxng/duckduckgo），默认关； 开关经 worker search_env 透传进 MCP config env。 - HttpClient 超时/DNS 失败改报显式错误（timeout after Ns / URL error），绝不把超时写成“无结果”。 - SearXNG 默认参数核实：代码无 SEARXNG_URL 硬编码默认值； SEARXNG_ENGINES 默认 bing,360search 已在基线；WEBSEARCH_TIMEOUT 默认 20 秒。主机 env 实测仅 SEARXNG_URL=http://searx.diante.me 一行，故默认 pin 生效；文档 https 漂移改回 http（主机 nginx 仅听 80、无 443 vhost，https 在 TLS 层即失败）。 - 旧测试更新原因：test_websearch_mcp 原断言默认 auto 为 perplexity 优先，与新冻结语义直接冲突，故改为断言冻结后 的 searxng -> duckduckgo 回退 + 不泄漏 key；旧 perplexity -> exa 回退断言移至解冻开关测试（WEBSEARCH_ENABLE_PAID=1 下）。 未删断言、未加 skip、未放宽门禁。 - 文档另删 WEBSEARCH_PROVIDERS 说法：代码从未实现该变量。 - 未推、未建 PR、未合并、未部署、未改主机 env/nginx/filter/ 证书、未做付费后端 live 验证、未创建生产任务。 --- cloud/tests/test_websearch_mcp.py | 34 ++- cloud/tests/test_websearch_paid_freeze.py | 280 ++++++++++++++++++++++ cloud/tests/test_websearch_worker.py | 16 +- cloud/websearch.py | 74 +++++- cloud/websearch_mcp.py | 16 +- cloud/worker.py | 6 + docs/web-search-mcp.md | 109 ++++++--- 7 files changed, 492 insertions(+), 43 deletions(-) create mode 100644 cloud/tests/test_websearch_paid_freeze.py diff --git a/cloud/tests/test_websearch_mcp.py b/cloud/tests/test_websearch_mcp.py index 36673de..a5ed5ed 100644 --- a/cloud/tests/test_websearch_mcp.py +++ b/cloud/tests/test_websearch_mcp.py @@ -67,18 +67,38 @@ def _exa_body(): class ChainTestCase(unittest.TestCase): - def test_fallback_perplexity_to_exa_and_key_not_leaked(self): + def test_default_auto_frozen_falls_back_searxng_to_duckduckgo_and_key_not_leaked(self): + # Paid-freeze update of the old perplexity -> exa fallback + # test: default auto no longer contains any paid backend, + # even when all four paid keys are set, so the fallback + # that still exists by default is searxng -> duckduckgo + # and no paid URL is ever touched. The old perplexity -> + # exa fallback is preserved, but only under + # WEBSEARCH_ENABLE_PAID=1; see test_websearch_paid_freeze.py. + html = ( + 'DDG Title' + 'ddg snippet') http = FakeHttp([ - ("perplexity", {}, 500), - ("api.exa.ai", _exa_body(), 200), + ("searx", {}, 500), + ("duckduckgo", html, 200), ]) + env = dict(_FAKE_KEYS) + env["SEARXNG_URL"] = "http://searx.test" out = search("test query", provider="auto", - env=dict(_FAKE_KEYS), http=http) - self.assertEqual(out["provider"], "exa") + env=env, http=http) + self.assertEqual(out["provider"], "duckduckgo") self.assertEqual(out["results"][0]["url"], - "https://example.com/exa") - self.assertEqual(out["attempted"][0]["provider"], "perplexity") + "https://example.com/ddg") + self.assertEqual(out["attempted"][0]["provider"], "searxng") self.assertFalse(out["attempted"][0]["ok"]) + self.assertEqual( + [a["provider"] for a in out["attempted"]], + ["searxng", "duckduckgo"]) + for _method, url, _payload in http.seen: + for paid in ("perplexity", "exa.ai", "tavily", + "search.brave.com"): + self.assertNotIn(paid, url) blob = json.dumps(out) for secret in _FAKE_KEYS.values(): self.assertNotIn(secret, blob) diff --git a/cloud/tests/test_websearch_paid_freeze.py b/cloud/tests/test_websearch_paid_freeze.py new file mode 100644 index 0000000..77a1553 --- /dev/null +++ b/cloud/tests/test_websearch_paid_freeze.py @@ -0,0 +1,280 @@ +"""Paid-freeze tests: default auto is free-only, pin still works, +unfreeze is explicit (WEBSEARCH_ENABLE_PAID=1, exact value "1"). + +No network, no real keys: every HTTP call goes through a FakeHttp, +and the fake key values are deliberately distinctive so a leak into +an error string or a tool result would be caught by assertion. +""" + +import json +import os +import socket +import sys +import unittest +import urllib.error +from pathlib import Path +from unittest import mock + +try: + from cloud.websearch import ( + DEFAULT_CHAIN, DEFAULT_FREE_CHAIN, HttpClient, WebSearchError, + available_providers, paid_search_enabled, + resolve_provider_chain, search) + from cloud.websearch_mcp import handle_request +except ImportError: + sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + from cloud.websearch import ( + DEFAULT_CHAIN, DEFAULT_FREE_CHAIN, HttpClient, WebSearchError, + available_providers, paid_search_enabled, + resolve_provider_chain, search) + from cloud.websearch_mcp import handle_request + +_FAKE_KEYS = { + "PERPLEXITY_API_KEY": "pplx-SECRET-111", + "EXA_API_KEY": "exa-SECRET-222", + "TAVILY_API_KEY": "tvly-SECRET-333", + "BRAVE_API_KEY": "brave-SECRET-444", +} + +_DDG_HTML = ( + 'DDG Title' + 'ddg snippet') + + +class FakeHttp: + """post_json/get keyed by URL substring; failures raise like HTTP.""" + + def __init__(self, routes): + self.routes = routes + self.seen = [] + + def post_json(self, url, headers, payload): + self.seen.append(("POST", url, payload)) + return self._route(url) + + def get(self, url, headers=None): + self.seen.append(("GET", url, headers)) + return self._route(url) + + def _route(self, url): + for needle, body, status in self.routes: + if needle in url: + if status != 200: + raise WebSearchError("HTTP %s" % status) + if isinstance(body, dict): + return 200, json.dumps(body).encode(), "application/json" + return 200, body.encode(), "text/html" + raise WebSearchError("HTTP 404") + + +def _searxng_body(): + return {"results": [ + {"title": "SearXNG Hit", "url": "https://example.com/sx", + "content": "searxng snippet"}]} + + +def _paid_env(**extra): + env = dict(_FAKE_KEYS) + env["SEARXNG_URL"] = "http://searx.test" + env.update(extra) + return env + + +def _paid_hosts_seen(http): + hosts = [] + for _method, url, _payload in http.seen: + for paid in ("perplexity", "exa.ai", "tavily", + "search.brave.com"): + if paid in url: + hosts.append(paid) + return hosts + + +class FrozenDefaultTestCase(unittest.TestCase): + def test_default_auto_excludes_paid_even_with_all_keys(self): + env = _paid_env() + self.assertEqual(available_providers(env), + ["searxng", "duckduckgo"]) + self.assertEqual(resolve_provider_chain("auto", env), + ["searxng", "duckduckgo"]) + http = FakeHttp([("searx", _searxng_body(), 200)]) + out = search("q", provider="auto", env=env, http=http) + self.assertEqual(out["provider"], "searxng") + self.assertEqual(_paid_hosts_seen(http), []) + + def test_default_auto_without_searxng_url_is_duckduckgo_only(self): + env = dict(_FAKE_KEYS) + self.assertEqual(resolve_provider_chain("auto", env), + ["duckduckgo"]) + http = FakeHttp([("duckduckgo", _DDG_HTML, 200)]) + out = search("q", provider="auto", env=env, http=http) + self.assertEqual(out["provider"], "duckduckgo") + self.assertEqual(_paid_hosts_seen(http), []) + + def test_searxng_failure_never_falls_back_to_paid(self): + http = FakeHttp([("searx", {}, 500), + ("duckduckgo", "challenge", 200)]) + with self.assertRaises(WebSearchError) as ctx: + search("q", provider="auto", env=_paid_env(), http=http) + self.assertEqual( + [a["provider"] for a in ctx.exception.attempted], + ["searxng", "duckduckgo"]) + self.assertEqual(_paid_hosts_seen(http), []) + # No fabricated result: the error names both free stages. + self.assertIn("searxng", str(ctx.exception)) + self.assertIn("duckduckgo", str(ctx.exception)) + + def test_duckduckgo_fallback_failure_reports_clearly(self): + # DuckDuckGo under the VM/host egress returns a 202-style + # robot-challenge page: the parser yields no results, which + # must surface as an honest "no results" failure for that + # stage, never as a fabricated hit. + http = FakeHttp([("searx", {}, 500), + ("duckduckgo", "anomaly", 200)]) + with self.assertRaises(WebSearchError) as ctx: + search("q", provider="auto", env=_paid_env(), http=http) + attempted = {a["provider"]: a for a in ctx.exception.attempted} + self.assertEqual(attempted["duckduckgo"]["error"], "no results") + self.assertFalse(attempted["duckduckgo"]["ok"]) + for secret in _FAKE_KEYS.values(): + self.assertNotIn(secret, str(ctx.exception)) + + +class ExplicitPinTestCase(unittest.TestCase): + def test_explicit_pin_paid_still_works_while_frozen(self): + http = FakeHttp([ + ("api.tavily.com", + {"results": [{"title": "T", "url": "https://example.com/t", + "content": "tavily content"}]}, 200), + ]) + out = search("q", provider="tavily", env=_paid_env(), http=http) + self.assertEqual(out["provider"], "tavily") + self.assertEqual(out["results"][0]["snippet"], "tavily content") + + def test_explicit_pin_paid_failure_does_not_fall_back(self): + http = FakeHttp([("perplexity", {}, 500), + ("api.exa.ai", {"results": [ + {"title": "E", "url": "https://example.com/e", + "text": "x"}]}, 200)]) + with self.assertRaises(WebSearchError): + search("q", provider="perplexity", env=_paid_env(), http=http) + self.assertEqual(_paid_hosts_seen(http), ["perplexity"]) + + def test_frozen_default_applies_to_scope_code_too(self): + body = {"results": [ + {"title": "GH", "url": "https://github.com/a/b", + "content": "code hit"}, + {"title": "Other", "url": "https://example.com/other", + "content": "off-scope"}]} + http = FakeHttp([("searx", body, 200)]) + out = search("q", provider="auto", env=_paid_env(), + http=http, scope="code") + self.assertEqual(out["provider"], "searxng") + self.assertEqual( + [r["url"] for r in out["results"]], ["https://github.com/a/b"]) + self.assertEqual(_paid_hosts_seen(http), []) + + +class UnfreezeSwitchTestCase(unittest.TestCase): + def test_switch_off_by_default_and_non_1_values_do_not_unfreeze(self): + self.assertFalse(paid_search_enabled({})) + for value in ("0", "true", "yes", "", "on"): + env = _paid_env(WEBSEARCH_ENABLE_PAID=value) + self.assertFalse(paid_search_enabled(env), value) + self.assertEqual(resolve_provider_chain("auto", env), + ["searxng", "duckduckgo"]) + + def test_switch_on_restores_legacy_chain_and_fallback(self): + env = _paid_env(WEBSEARCH_ENABLE_PAID="1") + self.assertTrue(paid_search_enabled(env)) + self.assertEqual(resolve_provider_chain("auto", env), + list(DEFAULT_CHAIN)) + # The pre-freeze perplexity -> exa fallback, preserved + # verbatim under the explicit switch (plus the no-leak + # assertion from the old default-chain test). + http = FakeHttp([ + ("perplexity", {}, 500), + ("api.exa.ai", {"results": [ + {"title": "Exa Hit", "url": "https://example.com/exa", + "text": "exa snippet", "score": 0.9}]}, 200), + ]) + out = search("test query", provider="auto", env=env, http=http) + self.assertEqual(out["provider"], "exa") + self.assertEqual(out["results"][0]["url"], + "https://example.com/exa") + self.assertEqual(out["attempted"][0]["provider"], "perplexity") + self.assertFalse(out["attempted"][0]["ok"]) + blob = json.dumps(out) + for secret in _FAKE_KEYS.values(): + self.assertNotIn(secret, blob) + + def test_constants_state_the_contract(self): + self.assertEqual(DEFAULT_FREE_CHAIN, ("searxng", "duckduckgo")) + self.assertEqual( + DEFAULT_CHAIN, + ("perplexity", "exa", "tavily", "brave", "searxng", + "duckduckgo")) + + +class TimeoutClarityTestCase(unittest.TestCase): + def test_http_client_maps_timeout_to_explicit_error(self): + client = HttpClient(timeout=20) + with mock.patch("urllib.request.urlopen", + side_effect=socket.timeout("timed out")): + with self.assertRaises(WebSearchError) as ctx: + client.get("http://searx.test/search?q=x") + self.assertIn("timeout after 20s", str(ctx.exception)) + + with mock.patch( + "urllib.request.urlopen", + side_effect=urllib.error.URLError( + socket.timeout("timed out"))): + with self.assertRaises(WebSearchError) as ctx: + client.get("http://searx.test/search?q=x") + self.assertIn("timeout after 20s", str(ctx.exception)) + + def test_http_client_does_not_relabel_dns_failure_as_timeout(self): + client = HttpClient(timeout=20) + with mock.patch( + "urllib.request.urlopen", + side_effect=urllib.error.URLError( + "Temporary failure in name resolution")): + with self.assertRaises(WebSearchError) as ctx: + client.get("http://searx.test/search?q=x") + self.assertNotIn("timeout", str(ctx.exception)) + self.assertIn("name resolution", str(ctx.exception)) + + def test_search_reports_timeout_as_error_not_no_results(self): + class TimeoutHttp(FakeHttp): + def get(self, url, headers=None): + self.seen.append(("GET", url, headers)) + if "searx" in url: + raise WebSearchError( + "timeout after 20s waiting for HTTP response") + return self._route(url) + + http = TimeoutHttp([("duckduckgo", _DDG_HTML, 200)]) + out = search("q", provider="auto", env=_paid_env(), http=http) + self.assertEqual(out["provider"], "duckduckgo") + attempted = {a["provider"]: a for a in out["attempted"]} + self.assertIn("timeout after 20s", + attempted["searxng"]["error"]) + self.assertNotEqual(attempted["searxng"]["error"], "no results") + + +class McpDescriptionTestCase(unittest.TestCase): + def test_tools_list_describes_frozen_default_and_unfreeze(self): + resp = handle_request( + {"jsonrpc": "2.0", "id": 1, "method": "tools/list"}) + tool = [t for t in resp["result"]["tools"] + if t["name"] == "web_search"][0] + desc = tool["description"] + self.assertIn("SearXNG", desc) + self.assertIn("DuckDuckGo", desc) + self.assertIn("WEBSEARCH_ENABLE_PAID=1", desc) + self.assertIn("paid", desc.lower()) + + +if __name__ == "__main__": + unittest.main() diff --git a/cloud/tests/test_websearch_worker.py b/cloud/tests/test_websearch_worker.py index 90d0956..756aa92 100644 --- a/cloud/tests/test_websearch_worker.py +++ b/cloud/tests/test_websearch_worker.py @@ -124,10 +124,24 @@ class DockerEnvTestCase(unittest.TestCase): self.assertIsNone(err) self.assertEqual(env.get("SEARXNG_URL"), "http://searx.diante.me") + def test_paid_unfreeze_switch_passes_into_container_env(self): + # The freeze switch must reach the MCP process (which reads + # only config.toml env, never worker env); without this + # passthrough, WEBSEARCH_ENABLE_PAID=1 on the host would + # silently leave default auto frozen inside containers. + from unittest import mock + with mock.patch.dict(os.environ, + {"WEBSEARCH_ENABLE_PAID": "1"}): + ok, _, _, env, err = self._docker_context() + self.assertTrue(ok) + self.assertIsNone(err) + self.assertEqual(env.get("WEBSEARCH_ENABLE_PAID"), "1") + def test_no_search_env_means_no_extra_env(self): from unittest import mock with mock.patch.dict(os.environ, {}, clear=False): - for k in ("SEARXNG_URL", "PERPLEXITY_API_KEY", "EXA_API_KEY", + for k in ("SEARXNG_URL", "WEBSEARCH_ENABLE_PAID", + "PERPLEXITY_API_KEY", "EXA_API_KEY", "TAVILY_API_KEY", "BRAVE_API_KEY"): os.environ.pop(k, None) ok, _, _, env, err = self._docker_context() diff --git a/cloud/websearch.py b/cloud/websearch.py index e42a72a..f1a3288 100644 --- a/cloud/websearch.py +++ b/cloud/websearch.py @@ -21,6 +21,16 @@ Design notes (what is copied from oh-my-pi, and what is not): here is a small stdlib HTML parser: script/style stripped, text collapsed, long pages truncated head+tail like Hermes' deterministic budget (default 15000 chars, 75% head / 25% tail, ``[TRUNCATED]``). +- FROZEN default chain (paid freeze): the default ``auto`` chain + contains only the free backends, SearXNG then DuckDuckGo, even + when paid keys happen to be set -- a SearXNG failure never falls + back to a paid backend, so default use can never incur paid cost. + The four paid backends (Perplexity, Exa, Tavily, Brave) keep their + code, tests and docs, and still run when explicitly pinned + (provider="tavily" etc.), which is a deliberate override, not a + default. Only WEBSEARCH_ENABLE_PAID=1 (exact value "1", default + off) restores the old default chain; unfreezing is an explicit act + and may incur paid cost, never a silent default. Security notes: - API keys come only from the process environment, never from @@ -57,6 +67,9 @@ from html.parser import HTMLParser __all__ = [ "DEFAULT_CHAIN", + "DEFAULT_FREE_CHAIN", + "PAID_PROVIDERS", + "paid_search_enabled", "CODE_SEARCH_DOMAINS", "WebSearchError", "SearchResult", @@ -72,6 +85,12 @@ __all__ = [ DEFAULT_CHAIN = ("perplexity", "exa", "tavily", "brave", "searxng", "duckduckgo") +# Paid backends, frozen out of the default auto chain (see the module +# docstring). DEFAULT_CHAIN keeps the legacy full order so the +# unfrozen chain is exactly the old one; DEFAULT_FREE_CHAIN is what +# default auto actually walks while frozen. +PAID_PROVIDERS = ("perplexity", "exa", "tavily", "brave") +DEFAULT_FREE_CHAIN = ("searxng", "duckduckgo") DEFAULT_FETCH_CHAR_LIMIT = 15000 # Programming-only search scope (``search(..., scope="code")``). @@ -246,6 +265,27 @@ class HttpClient: raise WebSearchError( "HTTP %s" % exc.code) from _ProviderHttpError( exc.code, body) + except (socket.timeout, TimeoutError) as exc: + # A timeout is a timeout, never "no results": the caller + # (search's attempted entry, named by provider) must be + # able to tell which stage timed out, and a timed-out + # SearXNG must not be rewritten as an empty answer. + # HTTPError is a URLError subclass, so this branch must + # stay after the HTTPError branch above. + raise WebSearchError( + "timeout after %ss waiting for HTTP response" + % self.timeout) from exc + except urllib.error.URLError as exc: + reason = getattr(exc, "reason", None) + if isinstance(reason, (socket.timeout, TimeoutError)): + raise WebSearchError( + "timeout after %ss waiting for HTTP response" + % self.timeout) from exc + # DNS / connection failures keep their own message; + # they are not timeouts and must not be relabelled. + raise WebSearchError( + "URL error: %s" % reason if reason else "URL error" + ) from exc def get_text(self, url, headers=None): status, body, _ctype = self.get(url, headers) @@ -534,11 +574,32 @@ _PROVIDERS = { } -def available_providers(env=None): - """Providers usable right now: keyed ones with a key, plus free ones.""" +def paid_search_enabled(env=None): + """True only when the paid-unfreeze switch is explicitly on. + + The switch is WEBSEARCH_ENABLE_PAID, and only the exact value + "1" counts (same strict convention as WEBSEARCH_ALLOW_PRIVATE_URLS + elsewhere in this module): any other value, including "true" or + "yes", leaves the paid backends frozen out of default auto. + Explicit pinning is unaffected by this switch either way. + """ env = env if env is not None else os.environ + return env.get("WEBSEARCH_ENABLE_PAID") == "1" + + +def available_providers(env=None): + """Providers usable right now, under the paid-freeze rule. + + Frozen (default): only the free chain -- SearXNG when its URL + is set, plus DuckDuckGo -- even when paid keys are present. + Unfrozen (WEBSEARCH_ENABLE_PAID=1): the legacy full chain, + keyed backends only when their key is set, as before. + """ + env = env if env is not None else os.environ + chain = (DEFAULT_CHAIN if paid_search_enabled(env) + else DEFAULT_FREE_CHAIN) out = [] - for name in DEFAULT_CHAIN: + for name in chain: if name in _FREE_PROVIDERS: if name == "searxng" and not env.get("SEARXNG_URL"): continue @@ -552,8 +613,11 @@ def resolve_provider_chain(provider="auto", env=None): """Resolve the chain oh-my-pi style: auto walks, pinned does not. ``provider`` may be "auto" (default chain, filtered to what is - configured), a single name (that provider only, even if it fails), - or a comma-separated list (that order only). Unknown names raise + configured -- frozen by default to the free chain, SearXNG then + DuckDuckGo, unless WEBSEARCH_ENABLE_PAID=1 restores the legacy + full chain), a single name (that provider only, even if it fails + -- pinning a paid backend bypasses the freeze, deliberately), or + a comma-separated list (that order only). Unknown names raise ValueError before any HTTP happens. """ env = env if env is not None else os.environ diff --git a/cloud/websearch_mcp.py b/cloud/websearch_mcp.py index 7c786c2..280d312 100644 --- a/cloud/websearch_mcp.py +++ b/cloud/websearch_mcp.py @@ -35,9 +35,19 @@ _TOOLS = [ "Search the web and return ranked results with title, URL " "and snippet, plus a synthesised answer when the provider " "supplies one. Use web_fetch to read specific pages. " - "provider may be 'auto' (walk Perplexity, Exa, Tavily, " - "Brave, then keyless SearXNG / DuckDuckGo, falling back " - "on failure) or a single provider name to pin it. " + "provider may be 'auto' (frozen default: keyless SearXNG " + "first, then DuckDuckGo as the free fallback, falling " + "back on failure; the paid backends Perplexity, Exa, " + "Tavily and Brave are NOT in the default chain even " + "when their keys are set, and a SearXNG failure never " + "falls back to a paid backend, so default use never " + "incurs paid cost) or a single provider name to pin it " + "(pinning a paid backend still calls it: that is an " + "explicit override, not a default). Set " + "WEBSEARCH_ENABLE_PAID=1 to restore the old default " + "chain (Perplexity, Exa, Tavily, Brave, SearXNG, " + "DuckDuckGo); unfreezing is explicit and may incur " + "paid cost. " "scope 'code' restricts results to programming sources " "only (official language docs, GitHub, package " "registries, Stack Overflow, CVE/NVD); anything else " diff --git a/cloud/worker.py b/cloud/worker.py index 805f865..564cc56 100644 --- a/cloud/worker.py +++ b/cloud/worker.py @@ -3188,9 +3188,15 @@ class Worker: # auto 链会跳过 SearXNG、只剩 DuckDuckGo（实弹 e2e 已证实）。 # 只透传已设置的项，值绝不进日志；付费 key 与网关 key 一样 # 走 extra_env（容器内进程本就需要它调 API）。 + # WEBSEARCH_ENABLE_PAID rides along so the paid-freeze + # switch set on the host actually reaches the MCP process + # (which reads only config.toml env, never worker env). + # Default is frozen; the switch is honoured only when set + # to exactly "1" (see websearch.paid_search_enabled). search_env = { k: os.environ[k] for k in ("SEARXNG_URL", "SEARXNG_ENGINES", + "WEBSEARCH_ENABLE_PAID", "PERPLEXITY_API_KEY", "EXA_API_KEY", "TAVILY_API_KEY", "BRAVE_API_KEY") if os.environ.get(k) diff --git a/docs/web-search-mcp.md b/docs/web-search-mcp.md index 0ec0759..cf47dc4 100644 --- a/docs/web-search-mcp.md +++ b/docs/web-search-mcp.md @@ -20,6 +20,11 @@ pi 的工具），只复用它的模式：统一工具 + provider 抽象 + - `cloud/tests/test_code_search_ext.py`：任务级 `allowed_hosts` 扩展的单测（指定 / 未指定 / 点边界 / 空值与非法值 / Perplexity 超 20 域 / MCP 透传）。 +- `cloud/tests/test_websearch_paid_freeze.py`：付费冻结单测 + （默认 auto 无付费后端、SearXNG 失败不回落付费、显式 + pin 仍工作、解冻开关关时不变 / 开时恢复旧链、 + DuckDuckGo 兜底失败报错清晰、超时报错清晰；全部 + mock HTTP，不联网、不需要真实 key）。 - `cloud/tests/test_js_heavy_extract.py`：JS 重型页面 抽取评估的 mock 单测（直接抓取壳-only、provider 抽取 补与不补、回退语义；live SearXNG 烟囱默认跳过）。 @@ -39,7 +44,48 @@ pi 的工具），只复用它的模式：统一工具 + provider 抽象 + 超长按头 75% + 尾 25% 截断并标 `[TRUNCATED]`；只接受 http(s)，拒绝 localhost 与字面私有 IP；抽取与直接抓取都失败时报错，不伪造页面。 -## Provider 链（auto 顺序） +## Provider 链：默认冻结，显式 pin 与显式解冻除外 + +冻结语义（本轨起，默认 `auto` 只走免费路线，顺序固定）： + +1. `searxng`：`GET $SEARXNG_URL/search?format=json`，需自托管 + 实例且 JSON 格式已打开。默认参数：`engines=bing,360search` + （2026-10-30 主机实测：默认 general 类别 20.0s 撞超时， + pinned 两引擎 0.47s；`SEARXNG_ENGINES` 可覆盖，空值恢复 + 不 pin）。`SEARXNG_URL` 在代码里无默认值：未配时该后端 + 直接报 `searxng not configured (SEARXNG_URL unset)`，链 + 只剩 DuckDuckGo；生产值 `http://searx.diante.me` 来自主 + 机 env（见“生产接线”），不在代码里写死；主机 env 实测 + 仅有 `SEARXNG_URL` 这一行、无 `SEARXNG_ENGINES` 行， + 所以默认 pin（bing,360search）生效。注意是 http 不是 + https：主机 nginx 只听 80、无 443 vhost，https 会在 TLS + 层证书主机名不匹配失败，根本到不了 SearXNG。 +2. `duckduckgo`：解析 `https://html.duckduckgo.com/html/`， + 免 key 兜底，布局漂移时只会返回空结果并继续走链， + 不会把工具打崩；但它在两种出口下都不可作可用兜底： + 本 VM 直连回 202 + 机器人挑战页、0 条，主机 Tinyproxy + 路径约 25 秒超时。兜底失败时记为 `no results` / 超时并 + 如实报错，不伪造结果；因此 SearXNG 挂时冻结默认链可能 + 以诚实失败告终，这是已知边界，不是静默降级。 + +冻结保证：即使环境里配了付费 key，默认 auto 也绝不调用 +Perplexity / Exa / Tavily / Brave、绝不产生付费花费； +SearXNG 失败后默认也不回落到付费后端，只走 DuckDuckGo。 +付费后端的代码、测试、文档一律保留，未删除。 + +显式 pin 仍工作：`provider="tavily"` / `"exa"` / +`"perplexity"` / `"brave"` 显式指定时照常调用，这是用户 +有意的覆盖，不是默认行为；pin 失败不回退（与冻结前一致）。 + +解冻开关：`WEBSEARCH_ENABLE_PAID=1`（且仅精确值 `1`， +`true` / `yes` / `0` 都不算开，默认关）时，默认 auto +恢复旧链，顺序为 Perplexity → Exa → Tavily → Brave → +SearXNG → DuckDuckGo（旧链里只试有 key 的付费后端 + +已配 URL 的 SearXNG + DuckDuckGo）。解冻是显式动作， +会产生付费花费，绝不能被默认链静默触发；开关走 worker +环境透传进 MCP config env（见“生产接线”）。 + +旧链各后端（解冻后 / 显式 pin 时）的请求形态不变： 1. `perplexity`：`POST https://api.perplexity.ai/chat/completions`， 模型 `sonar`，`Authorization: Bearer $PERPLEXITY_API_KEY`。 @@ -53,18 +99,16 @@ pi 的工具），只复用它的模式：统一工具 + provider 抽象 + 结果，总结留给模型）。 4. `brave`：`GET https://api.search.brave.com/res/v1/web/search`， 头 `X-Subscription-Token`，`extra_snippets: true`。 -5. `searxng`：`GET $SEARXNG_URL/search?format=json`，需自托管 - 实例且 JSON 格式已打开。 -6. `duckduckgo`：解析 `https://html.duckduckgo.com/html/`， - 免 key 兜底，布局漂移时只会返回空结果并继续走链， - 不会把工具打崩。 -规则：auto 只试已配置的后端（有 key 的付费后端 + 已配 -URL 的 SearXNG + DuckDuckGo）；任一后端 HTTP 失败或 -0 结果都继续下一个；全部失败返回各后端错误（不含 key）。 -`provider` pin 单个名字时不回退，失败即报错。 -可用 `WEBSEARCH_PROVIDERS`（逗号列表）改默认链顺序， -`WEBSEARCH_TIMEOUT`（秒，默认 20）改单次超时。 +规则：任一后端 HTTP 失败或 0 结果都继续下一个；全部失败 +返回各后端错误（不含 key）。超时失败报 +`timeout after Ns waiting for HTTP response`（N 为 +`WEBSEARCH_TIMEOUT`，默认 20 秒），并记在该后端的 +attempted 条目里，绝不把超时写成“无结果”或空答案。 +`WEBSEARCH_TIMEOUT`（秒，默认 20）改单次超时。另注意： +旧文档曾写可用 `WEBSEARCH_PROVIDERS` 改默认链顺序， +代码里从未实现该变量，本轨已删此说法（不另新加该变量， +保持改动最小）。 ## 接入 codex（测试项目做法） @@ -97,15 +141,19 @@ Docker 租户必须传容器内路径 - egress 白名单：`deploy/egress/tinyproxy-filter` 已加 Perplexity、Exa、Tavily、Brave、DuckDuckGo 与 `searx.diante.me`（精确匹配，不用 diante.me 通配，否则 - 任务经代理也能摸到 api.diante.me 控制面）。主机上须把 + 任务经代理也能摸到 api.diante.me 控制面；filter 行即 + `^searx\.diante\.me$`，本轨不改这行）。主机上须把 该文件拷到 `/etc/tinyproxy/tinyproxy-filter` 并重载 tinyproxy，重载后先验中转域名还通，再验新域名。 - 自托管 SearXNG（公开子域名路）：主机 Docker 跑 - `searxng/searxng`，经 nginx 反代到 `searx.diante.me` - （DNS A 记录 + certbot 证书），MCP 侧 - `SEARXNG_URL=https://searx.diante.me` 走 worker 进程 + `searxng/searxng`，经 nginx（仅听 80，无 443 vhost、无 + 证书）反代到 `searx.diante.me`，MCP 侧 + `SEARXNG_URL=http://searx.diante.me` 走 worker 进程 环境（`/etc/codex-app-server/env`）继承到 codex 再到 - MCP，不写进 config.toml。公开实例的代价是可能被外人 + MCP，不写进 config.toml。`WEBSEARCH_ENABLE_PAID` 同理 + 走 worker 的 search_env 透传进 `[mcp_servers.websearch.env]` + （worker 代码已含该键）；主机侧改 env 文件后需重启相关 + unit 才生效——那是部署动作，不在本轨。公开实例的代价是可能被外人 刷：SearXNG limiter 保持开启，JSON 格式是 MCP 要用的 唯一格式。未加域名进白名单之前，Docker 任务的搜索 会被代理 403 挡住（fail closed，工具报错，不会静默绕过）。 @@ -267,6 +315,10 @@ filter 行须用 POSIX BASIC 正则（`^\(.*\.\)*域名$`）、 - 本批只做代码 + 单测 + 测试租户目录的配置合并验证， 未推分支、未建 PR、未合并、未部署、未改生产 worker 与 egress 配置、未跑真实 codex 任务。 +- 付费冻结本轨同样只在本地提交里：生产默认链要等部署 + 新代码后才变；解冻开关在主机上默认仍是关（除非主机 + env 显式设为 `WEBSEARCH_ENABLE_PAID=1`，且那需要 + 主机侧改 env + 重启 unit，属部署动作）。 - DuckDuckGo 的 HTML 解析是兜底手段，布局一变就可能 0 结果；它存在的意义是“没 key 也能走通链路”，不是 长期依赖的搜索后端。实测（2026-10-28，VM 出口）： @@ -335,16 +387,19 @@ mock 补不了、也证明不了的部分： ### Live 对照（与 mock 分开写，未验证的不写成已验证） -- 无 key 的 SearXNG 路线：本轨在 VM 测试环境实测一次 - （`SEARXNG_URL=https://searx.diante.me`，`provider="searxng"`， - 产品自身 `search()` 代码）：失败，`URLError: Remote end - closed connection without response`。这是在本 VM 出口下 - 的一次结果，不能据此断言 SearXNG 服务本身挂了（生产 - 文档记载主机侧经代理 pinned 引擎曾 0.47s 通），live - 对照在本测试环境记为**未完成**，需在主机侧可达的环境 - 重测；测试文件里的 live 烟囱测试默认跳过，只在 - `WEBSEARCH_EVAL_LIVE_SEARXNG=1` 且配了 `SEARXNG_URL` - 时才跑。 +- 无 key 的 SearXNG 路线：旧 VM 测试曾用 + `SEARXNG_URL=https://searx.diante.me` 失败（`URLError: + Remote end closed connection without response`）——该失败 + 现已归因：https 是错的，主机 nginx 只听 80、无 443 + vhost，https 在 TLS 层证书主机名不匹配失败，根本没到 + SearXNG，不能据此断言服务挂了。2026-11-01 探针实测： + http 经同一 Tinyproxy 0.34 秒回 200、16 条结果，服务 + 活着；主机 env 仅 `SEARXNG_URL=http://searx.diante.me` + 一行、无 `SEARXNG_ENGINES`，默认 pin 生效。容器内真实 + 任务派发仍是 blocker（不由本轨创建生产任务），live + 对照在该项上仍记为**未完成**；测试文件里的 live 烟囱 + 测试默认跳过，只在 `WEBSEARCH_EVAL_LIVE_SEARXNG=1` + 且配了 `SEARXNG_URL` 时才跑。 - 付费后端 live 效果（Tavily extract / Exa contents / Perplexity / Brave 的真实 JS 渲染能力）：无 key，本轨 **一律未验证**，不编造效果，逐项列为 blocker（见回执），