feat: websearch MCP worker wiring + egress (public SearXNG route)
Worker --websearch-mcp (default off) seeds tenant codex-home with websearch MCP on _tenant_dirs (container path); old tenants merge on next task. Autoscaler unit gains the flag. Tinyproxy filter adds Perplexity/Exa/Tavily/Brave/DuckDuckGo + exact searx.diante.me (no diante.me wildcard, control plane stays out). Tests: test_websearch_worker 4 + websearch_mcp 12 + tenant 7 = 23 OK. Not pushed, not merged, not deployed; SearXNG host/DNS/nginx and tinyproxy reload are host steps.
This commit is contained in:
host committed
1 parent
075908fe58
commit
b72a2abae4
6 files changed
+209
-14
No files matched your search
@@ -3,6 +3,22 @@
|
||||
语义化版本见 `VERSION`（当前 0.1.0）。以下按日期倒序，只列已交付
|
||||
的大项，每项附实测证据；未建成的东西在“诚实边界”里明说，不夸大。
|
||||
|
||||
## 2026-10-28：网页搜索接 worker + egress（公开 SearXNG 路）
|
||||
|
||||
- `cloud/worker.py --websearch-mcp`（默认关）：开启后
|
||||
`_tenant_dirs` 播种 config.toml 之后自动播种 websearch
|
||||
MCP（容器内路径），老租户下一次任务自动合并；
|
||||
autoscaler unit 已加该 flag（主机上须单独应用 unit）。
|
||||
- `deploy/egress/tinyproxy-filter` 已加 Perplexity、Exa、
|
||||
Tavily、Brave、DuckDuckGo 与精确的 `searx.diante.me`
|
||||
（不用 diante.me 通配，防任务经代理摸到控制面）。
|
||||
- 单测：`cloud/tests/test_websearch_worker.py`（播种开/关、
|
||||
老租户合并、filter 的 BASIC 语法检查）与 websearch
|
||||
旧测、租户隔离旧测共 23 个全部通过。
|
||||
诚实边界：本批只本地代码，未推、未建 PR、未合并、
|
||||
未部署；SearXNG 主机部署、DNS、证书、nginx 与
|
||||
tinyproxy 重载均在主机步骤里，另见 docs/web-search-mcp.md。
|
||||
|
||||
## 2026-10-28：网页搜索 MCP v1（oh-my-pi 模式，Python 实现）
|
||||
|
||||
- 新增 `cloud/websearch.py` + `cloud/websearch_mcp.py`（纯标准库）：
|
||||
|
||||
@@ -0,0 +1,129 @@
|
||||
"""Worker wiring for web search MCP: tenant seeding + egress filter.
|
||||
|
||||
No Docker, no network: _tenant_dirs runs against temp dirs, and the
|
||||
egress check only reads the repo filter file.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
from cloud.server import init_db
|
||||
from cloud.worker import Worker
|
||||
except ImportError:
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||||
from cloud.server import init_db
|
||||
from cloud.worker import Worker
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
class WorkerWebsearchTestCase(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.db_path = os.path.join(self.tmp.name, "control.db")
|
||||
init_db(self.db_path)
|
||||
self.template_home = os.path.join(self.tmp.name, "tpl-home")
|
||||
os.makedirs(self.template_home)
|
||||
open(os.path.join(self.template_home, "config.toml"),
|
||||
"w").write('[model_provider.xcmapi]\n'
|
||||
'base_url="https://xcmapi.org/v1"\n')
|
||||
self.template_repo = os.path.join(self.tmp.name, "tpl-repo")
|
||||
os.makedirs(os.path.join(self.template_repo, ".git"))
|
||||
open(os.path.join(self.template_repo, "sample.py"),
|
||||
"w").write("def add(a, b):\n return a + b\n")
|
||||
|
||||
def tearDown(self):
|
||||
self.tmp.cleanup()
|
||||
|
||||
def _worker(self, **kwargs):
|
||||
kwargs.setdefault("env", {"XCM_API_KEY": "test-placeholder"})
|
||||
kwargs.setdefault("tenants_root",
|
||||
os.path.join(self.tmp.name, "tenants"))
|
||||
kwargs.setdefault("tenant_template_home", self.template_home)
|
||||
kwargs.setdefault("tenant_template_repo", self.template_repo)
|
||||
kwargs.setdefault("tenant_isolation", True)
|
||||
return Worker(self.db_path, "/fake/host.py", **kwargs)
|
||||
|
||||
def test_flag_off_is_old_behaviour(self):
|
||||
worker = self._worker()
|
||||
self.assertFalse(worker.websearch_mcp)
|
||||
dirs = worker._tenant_dirs("projA")
|
||||
config = open(os.path.join(dirs["codex_home"],
|
||||
"config.toml")).read()
|
||||
self.assertNotIn("[mcp_servers.websearch]", config)
|
||||
self.assertFalse(os.path.exists(
|
||||
os.path.join(dirs["codex_home"], "websearch_mcp.py")))
|
||||
|
||||
def test_flag_on_seeds_container_path_and_scripts(self):
|
||||
worker = self._worker(websearch_mcp=True)
|
||||
dirs = worker._tenant_dirs("projA")
|
||||
home = dirs["codex_home"]
|
||||
config = open(os.path.join(home, "config.toml")).read()
|
||||
self.assertIn("[mcp_servers.websearch]", config)
|
||||
# Container path, not the host temp path: this config is
|
||||
# read inside the task container, where the tenant home is
|
||||
# mounted at /var/lib/codex-app-server/codex-home.
|
||||
self.assertIn("/var/lib/codex-app-server/codex-home/"
|
||||
"websearch_mcp.py", config)
|
||||
self.assertNotIn(self.tmp.name, config)
|
||||
self.assertTrue(os.path.exists(
|
||||
os.path.join(home, "websearch.py")))
|
||||
self.assertTrue(os.path.exists(
|
||||
os.path.join(home, "websearch_mcp.py")))
|
||||
# The copied MCP script is the real one (not a stub).
|
||||
script = open(os.path.join(home, "websearch_mcp.py")).read()
|
||||
self.assertIn("web_search", script)
|
||||
self.assertIn("web_fetch", script)
|
||||
|
||||
def test_old_tenant_merges_on_next_task(self):
|
||||
worker = self._worker()
|
||||
dirs = worker._tenant_dirs("projA")
|
||||
config_path = os.path.join(dirs["codex_home"], "config.toml")
|
||||
self.assertNotIn("[mcp_servers.websearch]",
|
||||
open(config_path).read())
|
||||
# Flag flips on later (deploy enables it): next task's
|
||||
# _tenant_dirs must merge, not require a new project.
|
||||
worker2 = self._worker(websearch_mcp=True)
|
||||
worker2._tenant_dirs("projA")
|
||||
config = open(config_path).read()
|
||||
self.assertIn("[mcp_servers.websearch]", config)
|
||||
self.assertIn("xcmapi.org", config)
|
||||
self.assertEqual(config.count("[mcp_servers.websearch]"), 1)
|
||||
# Third call stays idempotent.
|
||||
worker2._tenant_dirs("projA")
|
||||
config = open(config_path).read()
|
||||
self.assertEqual(config.count("[mcp_servers.websearch]"), 1)
|
||||
|
||||
|
||||
class EgressFilterTestCase(unittest.TestCase):
|
||||
def test_search_domains_present_in_basic_form(self):
|
||||
lines = open(REPO_ROOT / "deploy" / "egress" /
|
||||
"tinyproxy-filter").read().splitlines()
|
||||
entries = [ln for ln in lines
|
||||
if ln and not ln.startswith("#")]
|
||||
# No extended-regex syntax in any entry: + ? | and bare
|
||||
# parens are literal in BASIC and would break the allowlist.
|
||||
for entry in entries:
|
||||
self.assertNotIn("+", entry)
|
||||
self.assertNotIn("?", entry)
|
||||
self.assertNotIn("|", entry)
|
||||
self.assertNotIn("(", entry.replace("\\(", ""))
|
||||
self.assertNotIn(")", entry.replace("\\)", ""))
|
||||
expected = [
|
||||
r"^\(.*\.\)*perplexity\.ai$",
|
||||
r"^\(.*\.\)*exa\.ai$",
|
||||
r"^\(.*\.\)*tavily\.com$",
|
||||
r"^\(.*\.\)*search\.brave\.com$",
|
||||
r"^\(.*\.\)*duckduckgo\.com$",
|
||||
r"^searx\.diante\.me$",
|
||||
]
|
||||
for want in expected:
|
||||
self.assertIn(want, entries, "missing filter entry: %s" % want)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
+30
-1
@@ -152,6 +152,7 @@ from cloud.peer_messages import deliver_peer_message as _deliver_peer_message
|
||||
from cloud.peer_messages import peer_message_prefix as _peer_prefix
|
||||
from cloud.peer_messages import row_get as _row_get
|
||||
from cloud.sub2api import ensure_user_gateway_key as _ensure_sub2api_key
|
||||
from cloud.websearch import ensure_codex_home_websearch as _ensure_websearch_mcp
|
||||
from cloud.sub2api import gateway_name as _gateway_name
|
||||
from cloud.sub2api import gateway_origin as _gateway_origin
|
||||
from cloud.sub2api import list_gateway_models as _list_gateway_models
|
||||
@@ -371,6 +372,7 @@ class Worker:
|
||||
tenant_isolation=False, tenants_root=None,
|
||||
tenant_template_home=None, tenant_template_repo=None,
|
||||
egress_proxy_port=0, claim_executor_only=False,
|
||||
websearch_mcp=False,
|
||||
host_name=None, pr_jobs_only=False,
|
||||
upload_dir=None,
|
||||
digest_interval_seconds=3600,
|
||||
@@ -506,6 +508,13 @@ class Worker:
|
||||
# 容器的 NULL 任务（那些任务在宿主上跑就丢了 Docker
|
||||
# 外层，违背“按任务二选一”的路由意图）。
|
||||
self.claim_executor_only = bool(claim_executor_only)
|
||||
# 网页搜索 MCP：默认关（老行为：租户 codex-home 只有
|
||||
# 模板 config.toml）。开启后 _tenant_dirs 在播种 config
|
||||
# 之后调用 websearch 的播种助手，把 websearch.py /
|
||||
# websearch_mcp.py 拷进租户 codex-home 并追加
|
||||
# [mcp_servers.websearch]（容器内路径，见 _tenant_dirs）。
|
||||
# 老租户下一次任务自动合并，已有 section 不重写。
|
||||
self.websearch_mcp = bool(websearch_mcp)
|
||||
# HA 阶段 1：本 worker 所在主机名。认领任务时写进
|
||||
# tasks.worktree_host，/pr 据此判断 worktree 在哪台主机。
|
||||
# 容器内 worker 的 socket.gethostname() 是容器名、不是宿主名，
|
||||
@@ -1788,6 +1797,19 @@ class Worker:
|
||||
" 会让 codex 去 api.openai.com 且无 auth，拒绝"
|
||||
" 派发）" % template)
|
||||
shutil.copyfile(template, home_config)
|
||||
if self.websearch_mcp:
|
||||
# Docker 租户任务的 codex-home 会原样挂载进容器，
|
||||
# 所以这里写容器内路径；宿主侧脚本文件由播种助手
|
||||
# 从运行中代码的 cloud/ 目录拷过来（生产即
|
||||
# /opt/codex-app-server/cloud/）。播种失败直接
|
||||
# 抛错、任务不派发——静默无搜索比 loud 失败更糟，
|
||||
# 尤其对已宣传“有搜索”的任务。
|
||||
_ensure_websearch_mcp(
|
||||
dirs["codex_home"],
|
||||
command_args=[
|
||||
"python3",
|
||||
"/var/lib/codex-app-server/codex-home/"
|
||||
"websearch_mcp.py"])
|
||||
if not os.path.exists(os.path.join(dirs["test_repo"], ".git")) \
|
||||
and not os.path.exists(dirs["test_repo"] + "/sample.py"):
|
||||
# 目标为空目录（刚 mkdir 出来的）才播种；已有内容
|
||||
@@ -4415,6 +4437,12 @@ def main(argv=None):
|
||||
" executor 为 NULL 的任务。宿主级沙箱"
|
||||
" worker 必须开，否则它会抢走本该进"
|
||||
" Docker 容器的 NULL 任务")
|
||||
parser.add_argument("--websearch-mcp", action="store_true",
|
||||
help="租户 codex-home 自动播种网页搜索 MCP："
|
||||
"_tenant_dirs 在播种 config.toml 后追加"
|
||||
" [mcp_servers.websearch] 并拷入"
|
||||
" websearch.py / websearch_mcp.py；"
|
||||
"默认关，老租户不受影响")
|
||||
parser.add_argument("--task-timeout", type=float, default=1800,
|
||||
help="每任务截止时间（秒，默认 1800）；0 表示不限时"
|
||||
"（与 host --turn-timeout 0 同约定，worker 级"
|
||||
@@ -4470,7 +4498,8 @@ def main(argv=None):
|
||||
tenant_template_home=args.tenant_template_home,
|
||||
tenant_template_repo=args.tenant_template_repo,
|
||||
egress_proxy_port=args.egress_proxy_port,
|
||||
claim_executor_only=args.claim_executor_only)
|
||||
claim_executor_only=args.claim_executor_only,
|
||||
websearch_mcp=args.websearch_mcp)
|
||||
# CLI 启动清理兜底（常驻与单次都做）：清掉上一轮残留容器。
|
||||
worker.cleanup_stale_containers()
|
||||
# CLI 起 worker 前先建表/迁移：全新库直接起循环不应因缺表而炸。
|
||||
|
||||
@@ -14,7 +14,7 @@ EnvironmentFile=/etc/codex-app-server/env
|
||||
# scaler 会按实例强制追加；--db 只给 scaler，scaler 会补给 worker。
|
||||
# 注意：启用 autoscaler 后必须停掉静态 codex-cloud-worker，
|
||||
# 否则两边同时派任务（scaler 只管理 autoscale-* 实例）。
|
||||
ExecStart=/usr/bin/python3 -m cloud.autoscaler --db /var/lib/codex-app-server/control.db --interval 30 --min-workers 1 --max-workers 2 --per-worker-capacity 2 --scale-down-after 3 --drain-timeout 60 --state-file /var/lib/codex-app-server/autoscaler-state.json --log-dir /var/lib/codex-app-server/autoscaler-logs -- --interval 10 --config /opt/codex-app-server/host-config-docker.json --executor docker --docker-image codex-app-server:latest --docker-user 1003:1004 --docker-cpus 2 --docker-memory 2g --docker-extra-mounts /opt/codex-app-server/work:/app/work,/var/lib/codex-app-server/codex-home:/var/lib/codex-app-server/codex-home,/var/lib/codex-app-server/repos:/var/lib/codex-app-server/repos,/var/lib/codex-app-server/test-repo:/var/lib/codex-app-server/test-repo --repos-dir /var/lib/codex-app-server/repos --tenant-isolation --tenants-root /var/lib/codex-app-server/tenants --tenant-template-home /var/lib/codex-app-server/codex-home --tenant-template-repo /var/lib/codex-app-server/test-repo --docker-security-opt apparmor=docker-codex --docker-pids-limit 512 --egress-proxy-port 8888 --docker-runtime runsc
|
||||
ExecStart=/usr/bin/python3 -m cloud.autoscaler --db /var/lib/codex-app-server/control.db --interval 30 --min-workers 1 --max-workers 2 --per-worker-capacity 2 --scale-down-after 3 --drain-timeout 60 --state-file /var/lib/codex-app-server/autoscaler-state.json --log-dir /var/lib/codex-app-server/autoscaler-logs -- --interval 10 --config /opt/codex-app-server/host-config-docker.json --executor docker --docker-image codex-app-server:latest --docker-user 1003:1004 --docker-cpus 2 --docker-memory 2g --docker-extra-mounts /opt/codex-app-server/work:/app/work,/var/lib/codex-app-server/codex-home:/var/lib/codex-app-server/codex-home,/var/lib/codex-app-server/repos:/var/lib/codex-app-server/repos,/var/lib/codex-app-server/test-repo:/var/lib/codex-app-server/test-repo --repos-dir /var/lib/codex-app-server/repos --tenant-isolation --tenants-root /var/lib/codex-app-server/tenants --tenant-template-home /var/lib/codex-app-server/codex-home --tenant-template-repo /var/lib/codex-app-server/test-repo --docker-security-opt apparmor=docker-codex --docker-pids-limit 512 --egress-proxy-port 8888 --docker-runtime runsc --websearch-mcp
|
||||
Restart=on-failure
|
||||
RestartSec=10
|
||||
# 停 scaler 时要等它把 worker 优雅排空（drain 60 秒 + 缓冲），
|
||||
|
||||
@@ -21,3 +21,13 @@
|
||||
^\(.*\.\)*pypi\.org$
|
||||
^\(.*\.\)*pythonhosted\.org$
|
||||
^\(.*\.\)*npmjs\.org$
|
||||
# Web search MCP v1 (see docs/web-search-mcp.md). Each line is the
|
||||
# API host's domain, subdomains included, in the same BASIC form.
|
||||
# searx.diante.me is exact-only on purpose: a \(.*\.\)*diante.me$
|
||||
# line would also let tasks reach api.diante.me (the control plane).
|
||||
^\(.*\.\)*perplexity\.ai$
|
||||
^\(.*\.\)*exa\.ai$
|
||||
^\(.*\.\)*tavily\.com$
|
||||
^\(.*\.\)*search\.brave\.com$
|
||||
^\(.*\.\)*duckduckgo\.com$
|
||||
^searx\.diante\.me$
|
||||
+23
-12
@@ -73,20 +73,31 @@ Docker 租户必须传容器内路径
|
||||
`/var/lib/codex-app-server/codex-home/websearch_mcp.py`，
|
||||
因为该目录正是租户 codex-home 的容器内挂载点。
|
||||
|
||||
## 生产接线前必须另做的事（本批未做）
|
||||
## 生产接线（worker + egress + 自托管 SearXNG）
|
||||
|
||||
- worker 自动播种：目前播种助手已写好并有单测，但没有
|
||||
接进 worker 的派发路径，新项目不会自动获得搜索工具；
|
||||
老项目（config.toml 已存在且无该 section）需要一次
|
||||
配置合并，不能只改模板。
|
||||
- egress 白名单：Docker 任务走 tinyproxy，只放行中转、
|
||||
GitHub、PyPI、npmjs；搜索要通必须把以下域名加进
|
||||
`deploy/egress/tinyproxy-filter` 并重载 tinyproxy：
|
||||
`api.perplexity.ai`、`api.exa.ai`、`api.tavily.com`、
|
||||
`api.search.brave.com`、`html.duckduckgo.com`，以及你
|
||||
自己的 SearXNG 实例域名。未加之前，Docker 任务的搜索
|
||||
- worker 自动播种：`cloud/worker.py --websearch-mcp`（默认关）。
|
||||
开启后 `_tenant_dirs` 在播种 config.toml 之后调播种助手，
|
||||
写容器内路径 `/var/lib/codex-app-server/codex-home/websearch_mcp.py`。
|
||||
老租户下一次任务自动合并（`_tenant_dirs` 每次派发都调到），
|
||||
已有 section 不重写。autoscaler unit 已加 `--websearch-mcp`，
|
||||
但 unit 文件不在 deploy tar 里，主机上须单独 scp +
|
||||
daemon-reload + restart autoscaler 才生效（见部署教训）。
|
||||
- egress 白名单：`deploy/egress/tinyproxy-filter` 已加
|
||||
Perplexity、Exa、Tavily、Brave、DuckDuckGo 与
|
||||
`searx.diante.me`（精确匹配，不用 diante.me 通配，否则
|
||||
任务经代理也能摸到 api.diante.me 控制面）。主机上须把
|
||||
该文件拷到 `/etc/tinyproxy/tinyproxy-filter` 并重载
|
||||
tinyproxy，重载后先验中转域名还通，再验新域名。
|
||||
- 自托管 SearXNG（公开子域名路）：主机 Docker 跑
|
||||
`searxng/searxng`，经 nginx 反代到 `searx.diante.me`
|
||||
（DNS A 记录 + certbot 证书），MCP 侧
|
||||
`SEARXNG_URL=https://searx.diante.me` 走 worker 进程
|
||||
环境（`/etc/codex-app-server/env`）继承到 codex 再到
|
||||
MCP，不写进 config.toml。公开实例的代价是可能被外人
|
||||
刷：SearXNG limiter 保持开启，JSON 格式是 MCP 要用的
|
||||
唯一格式。未加域名进白名单之前，Docker 任务的搜索
|
||||
会被代理 403 挡住（fail closed，工具报错，不会静默绕过）。
|
||||
- 付费后端实测：本批付费后端只以 mock 响应验证了请求
|
||||
- 付费后端实测：付费后端只以 mock 响应验证了请求
|
||||
字段与结果映射，未用真实 key 跑过；上生产前须用真实
|
||||
key 在测试项目跑一遍，再谈默认链顺序是否合理。
|
||||
|
||||
|
||||
Reference in new issue
Block a user