From b977a9dee81cb883e1ebaeb5ece156c36056ddb0 Mon Sep 17 00:00:00 2001 From: host Date: Fri, 30 Oct 2026 06:59:10 +0000 Subject: [PATCH] feat: sync shared-home MCP scripts on control-plane startup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 租户 config 的 mcp_servers.* 指向共享 home 路径，但宿主共享 home 里原先没有 MCP 脚本，宿主 app-server 启动 MCP 即文件 不存在、握手断掉，面板 mcp-servers-list 三服务器 toolsError 全是 handshake failed；14:43 的手工拷入能修一次，但下次 MCP 脚本改动后面板又会跑旧版。 本批只在服务端 main() 启动时同步一次：清单唯一定义在 cloud/shared_home_mcp.py（7 文件：playwright_mcp/browser/ remote、websearch_mcp/websearch、imagegen_mcp/imagegen，含 同目录兄弟模块），解析优先用 host-config-host.json 的 codex_home，兜底用 tenants_root 父目录/codex-home；缺失 拷入、不同刷新（原子替换）、相同不动；源缺失/目标不可写 只记 warning，绝不阻塞启动。create_server() 不做同步， 避免单测写向生产共享路径。 被否决的落点：worker _tenant_dirs（只在下个任务派发时跑， 面板要等下一个任务才更新，且不修共享路径本身）；deploy host_deploy.sh（会在 shell 里复制第二份清单，且重启不 部署时不刷新）；面板请求时同步（把读请求变成写请求， 失败会把面板读数变成 502 风险）。 --- cloud/server.py | 49 +++++++ cloud/shared_home_mcp.py | 139 ++++++++++++++++++ cloud/tests/test_shared_home_mcp.py | 213 ++++++++++++++++++++++++++++ 3 files changed, 401 insertions(+) create mode 100644 cloud/shared_home_mcp.py create mode 100644 cloud/tests/test_shared_home_mcp.py diff --git a/cloud/server.py b/cloud/server.py index f1b5dbe..df496ac 100644 --- a/cloud/server.py +++ b/cloud/server.py @@ -107,6 +107,7 @@ from cloud.wait_ops import poll_task_statuses as _wait_poll # noqa: F401 # /wait 本身走首变轮询（_wait_poll_first），见 _task_wait。 from cloud.wait_ops import report_handoff as _handoff_report from cloud.wait_ops import worktree_status as _wait_worktree_status +from cloud.shared_home_mcp import sync_shared_home_mcp as _sync_shared_home_mcp from cloud.automations import ensure_automations_tables as _auto_ensure from cloud.automations import list_automations as _auto_list from cloud.todos import create_todo as _todo_create @@ -425,6 +426,30 @@ def _local_worker_codex_home(): return None +def _resolve_shared_home_mcp(server=None): + """共享 home 解析：供服务端启动时的 MCP 脚本同步用。 + + 优先级（生产路径绝不硬编码在这里，全部来自已有解析）： + 1. _local_worker_codex_home()：读 host-config-host.json 的 + codex_home，生产即共享 home； + 2. 兜底：tenants_root 的父目录 + codex-home（tenants_root + 本身来自 server 属性 / CODEX_TENANTS_ROOT / 默认值）。 + 任一解析不出时返回 None，调用方跳过同步、不因此报错。 + 注意：本函数只解析路径，不做文件同步；文件清单与同步 + 语义只在 cloud/shared_home_mcp.py 里。 + """ + local = _local_worker_codex_home() + if local: + return local + try: + root = _tenants_root_for(server) + except Exception: + return None + if not root: + return None + return os.path.join(os.path.dirname(root), "codex-home") + + def _tenant_work_dir(server, project_id): """项目对应的租户工作目录：tenants_root/<安全项目名>/work， skills/list 的 cwds 过滤用它（不传 cwd 时 app-server 会 @@ -10309,6 +10334,30 @@ def main(argv=None): args = parser.parse_args(argv) server = create_server(args.db, args.host, args.port, backend=args.db_backend) + # 共享 home MCP 脚本同步：只在 main() 启动时做，不在 + # create_server() 里做——create_server 是单测入口，单测若 + # 在此同步会按 host-config-host.json 写向生产共享路径。 + # systemd 两个控制面都跑 python3 -m cloud.server，走的 + # 都是 main()，每次部署重启都会同步一次；同步失败只记 + # warning，绝不阻塞启动（sync 本身也不抛异常，外层再 + # 兜一层防解析等意外）。Docker 任务不受影响：容器内 + # 该共享路径是租户 home 的挂载，宿主补齐不进容器。 + try: + _shared_home = _resolve_shared_home_mcp(server) + if _shared_home: + _sync_result = _sync_shared_home_mcp(_shared_home) + if _sync_result.get("copied") or _sync_result.get("refreshed"): + _server_log.info( + "shared home MCP synced: copied=%s refreshed=%s", + _sync_result.get("copied"), + _sync_result.get("refreshed")) + if _sync_result.get("error"): + _server_log.warning( + "shared home MCP sync failed (non-fatal): %s", + _sync_result.get("error")) + except Exception as exc: + _server_log.warning( + "shared home MCP sync failed (non-fatal): %s", exc) start_rate_limit_sweeper(server.rate_limiter, args.db) print( "cloud-control-plane listening on %s:%d db=%s" diff --git a/cloud/shared_home_mcp.py b/cloud/shared_home_mcp.py new file mode 100644 index 0000000..737df89 --- /dev/null +++ b/cloud/shared_home_mcp.py @@ -0,0 +1,139 @@ +"""共享 codex-home 的 MCP 脚本自动补齐。 + +背景（2026-10-30 实测）：租户 config 的 ``mcp_servers.*`` 指向 +共享路径 ``/playwright_mcp.py`` 等，但宿主共享 home +里原先一个 MCP 脚本都没有——宿主 app-server 按该路径启动 MCP +时文件不存在、握手断掉，面板 ``mcp-servers-list`` 三服务器的 +``toolsError`` 全是 ``handshake failed``。Docker 任务不受影响， +是因为租户 home 被挂载到容器内那个共享路径，容器内能解析。 + +本模块是这 7 个文件清单的唯一定义处：服务端启动同步、 +测试、以及将来任何其它调用方都只从这里取清单，绝不在 +server/worker/deploy 三处各写一份 7 文件列表（清单漂移就是 +下一次“面板又跑旧版”的根源）。 + +语义与 playwright 播种一致，而不是 websearch/imagegen 的租户 +播种语义：共享 home 里的脚本是系统文件，不是租户数据，租户 +不会去改它，所以 +- 缺失则拷入； +- 内容不同则刷新（原子替换）并记入 ``refreshed``； +- 内容相同则不动，绝不重写 mtime。 +其它文件（config.toml、技能、租户数据）一律不动。 + +失败语义：本函数绝不抛异常阻塞调用方。源缺失、目标目录建 +不出、单个文件不可写时只记入返回值（调用方记 warning），其 +余文件照常同步。调用方（服务端启动）因此永远不会因为一次 +补齐失败而起不来。 +""" + +import os +import tempfile + +# 唯一清单：MCP 脚本与其同目录兄弟模块。只拷 *_mcp.py 不够， +# 因为 MCP 脚本 fallback 时从同目录 import 兄弟模块 +# （playwright_mcp -> playwright_browser/playwright_remote， +# websearch_mcp -> websearch，imagegen_mcp -> imagegen）。 +SHARED_HOME_MCP_FILES = ( + "playwright_mcp.py", + "playwright_browser.py", + "playwright_remote.py", + "websearch_mcp.py", + "websearch.py", + "imagegen_mcp.py", + "imagegen.py", +) + + +def _cloud_dir(): + """部署代码的 cloud/ 目录：同步源，绝不是租户目录。""" + return os.path.dirname(os.path.abspath(__file__)) + + +def _write_atomic(dst, data): + """同目录临时文件 + os.replace 原子替换，避免并发启动的 + 另一控制面实例读到半写文件。""" + directory = os.path.dirname(dst) or "." + fd, tmp = tempfile.mkstemp(prefix=".mcp-sync-", dir=directory) + try: + with os.fdopen(fd, "wb") as fh: + fh.write(data) + os.replace(tmp, dst) + except BaseException: + try: + os.unlink(tmp) + except OSError: + pass + raise + + +def sync_shared_home_mcp(shared_home, source_dir=None): + """把部署版 MCP 脚本同步到共享 home。 + + 参数： + - shared_home：目标目录（测试时传临时目录；生产由服务端 + 的解析函数给出，见 cloud/server.py 的 + ``_resolve_shared_home_mcp``）。 + - source_dir：源目录，缺省为本模块所在的 cloud/ 目录。 + + 返回 dict，键固定为 copied / refreshed / unchanged / + missing / error，调用方只看它、不捕获异常： + - copied：目标缺失、本次拷入的文件； + - refreshed：目标存在但内容不同、本次刷新的文件； + - unchanged：内容相同、本次未动的文件； + - missing：源目录里不存在的文件（跳过，不阻塞其它文件）； + - error：None 或把缺失/写失败拼成的一行文本，调用方记 + warning 即可，绝不因此阻塞启动/部署/派发。 + """ + copied = [] + refreshed = [] + unchanged = [] + missing = [] + errors = [] + if not shared_home: + return {"copied": copied, "refreshed": refreshed, + "unchanged": unchanged, "missing": missing, + "error": "no shared home resolved"} + source = os.path.abspath(source_dir) if source_dir else _cloud_dir() + try: + os.makedirs(shared_home, exist_ok=True) + except Exception as exc: # 目录建不出：只记错，不阻塞 + return {"copied": copied, "refreshed": refreshed, + "unchanged": unchanged, "missing": missing, + "error": "cannot create shared home %s: %s" + % (shared_home, exc)} + for name in SHARED_HOME_MCP_FILES: + src = os.path.join(source, name) + dst = os.path.join(shared_home, name) + try: + with open(src, "rb") as fh: + data = fh.read() + except OSError: + missing.append(name) + continue + except Exception as exc: + errors.append("%s: cannot read source: %s" % (name, exc)) + continue + try: + if os.path.isfile(dst): + with open(dst, "rb") as fh: + if fh.read() == data: + unchanged.append(name) + continue + _write_atomic(dst, data) + refreshed.append(name) + elif os.path.exists(dst): + # 目标是目录等非文件：不删、不覆盖，只记错。 + errors.append("%s: destination is not a file" % name) + else: + _write_atomic(dst, data) + copied.append(name) + except Exception as exc: + errors.append("%s: %s" % (name, exc)) + continue + parts = [] + if missing: + parts.append("missing source files: %s" % ",".join(missing)) + parts.extend(errors) + return {"copied": copied, "refreshed": refreshed, + "unchanged": unchanged, "missing": missing, + "error": "; ".join(parts) if parts else None} diff --git a/cloud/tests/test_shared_home_mcp.py b/cloud/tests/test_shared_home_mcp.py new file mode 100644 index 0000000..3adc323 --- /dev/null +++ b/cloud/tests/test_shared_home_mcp.py @@ -0,0 +1,213 @@ +"""Tests for shared-home MCP script sync (cloud/shared_home_mcp.py). + +No production path is ever touched here: every sync target is a +temp dir, and the server-resolution tests monkeypatch the resolver +inputs instead of reading the repo's host-config-host.json (which +points at the production shared home). + +Coverage: +- the 7-file list is the single source of truth (exact set); +- missing files are copied; +- differing files are refreshed, same files are untouched + (mtime unchanged); +- missing source files do not crash and are reported; +- an uncreatable target and a per-file unwritable destination + (a directory in place of a file, root-proof) do not crash and + do not block the other files; +- server resolution: local worker home wins, tenants-root + parent is the fallback; +- server main() actually calls the sync at startup and survives + a sync error (fake server, no real port, no real DB). +""" + +import os +import sys +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) + +from shared_home_mcp import SHARED_HOME_MCP_FILES # noqa: E402 +from shared_home_mcp import sync_shared_home_mcp # noqa: E402 + +try: + from cloud.server import _resolve_shared_home_mcp + from cloud.server import main as _server_main + _SERVER_OK = True +except ImportError: + sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + try: + from cloud.server import _resolve_shared_home_mcp + from cloud.server import main as _server_main + _SERVER_OK = True + except ImportError: + _resolve_shared_home_mcp = None + _server_main = None + _SERVER_OK = False + + +def _write_source(source_dir, contents): + for name, data in contents.items(): + with open(os.path.join(source_dir, name), "wb") as fh: + fh.write(data) + + +class SharedHomeMcpSyncTest(unittest.TestCase): + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.source = os.path.join(self._tmp.name, "cloud") + self.target = os.path.join(self._tmp.name, "shared-home") + os.makedirs(self.source) + self.contents = {name: ("# %s v2\n" % name).encode() + for name in SHARED_HOME_MCP_FILES} + + def tearDown(self): + self._tmp.cleanup() + + def test_file_list_is_exactly_the_seven(self): + self.assertEqual(len(SHARED_HOME_MCP_FILES), 7) + self.assertEqual(len(set(SHARED_HOME_MCP_FILES)), 7) + self.assertEqual(set(SHARED_HOME_MCP_FILES), { + "playwright_mcp.py", "playwright_browser.py", + "playwright_remote.py", + "websearch_mcp.py", "websearch.py", + "imagegen_mcp.py", "imagegen.py", + }) + + def test_missing_files_are_copied(self): + _write_source(self.source, self.contents) + result = sync_shared_home_mcp(self.target, self.source) + self.assertEqual(sorted(result["copied"]), + sorted(SHARED_HOME_MCP_FILES)) + self.assertEqual(result["refreshed"], []) + self.assertEqual(result["unchanged"], []) + self.assertEqual(result["missing"], []) + self.assertIsNone(result["error"]) + for name, data in self.contents.items(): + with open(os.path.join(self.target, name), "rb") as fh: + self.assertEqual(fh.read(), data) + + def test_differing_files_are_refreshed(self): + _write_source(self.source, self.contents) + os.makedirs(self.target) + for name in SHARED_HOME_MCP_FILES: + with open(os.path.join(self.target, name), "wb") as fh: + fh.write(b"# old\n") + result = sync_shared_home_mcp(self.target, self.source) + self.assertEqual(sorted(result["refreshed"]), + sorted(SHARED_HOME_MCP_FILES)) + self.assertEqual(result["copied"], []) + self.assertIsNone(result["error"]) + + def test_same_files_are_untouched(self): + _write_source(self.source, self.contents) + os.makedirs(self.target) + mtimes = {} + for name, data in self.contents.items(): + path = os.path.join(self.target, name) + with open(path, "wb") as fh: + fh.write(data) + os.utime(path, (1000000000, 1000000000)) + mtimes[name] = os.stat(path).st_mtime + result = sync_shared_home_mcp(self.target, self.source) + self.assertEqual(sorted(result["unchanged"]), + sorted(SHARED_HOME_MCP_FILES)) + self.assertEqual(result["copied"], []) + self.assertEqual(result["refreshed"], []) + for name, mtime in mtimes.items(): + self.assertEqual(os.stat(os.path.join(self.target, name)).st_mtime, + mtime) + + def test_missing_source_does_not_crash(self): + # Source dir exists but is empty: every file is missing. + result = sync_shared_home_mcp(self.target, self.source) + self.assertEqual(sorted(result["missing"]), + sorted(SHARED_HOME_MCP_FILES)) + self.assertEqual(result["copied"], []) + self.assertIsNotNone(result["error"]) + self.assertIn("missing source", result["error"]) + + def test_uncreatable_target_does_not_crash(self): + _write_source(self.source, self.contents) + # Target path is an existing file, not a directory: + # makedirs fails even as root, and must not raise. + blocker = os.path.join(self._tmp.name, "blocker") + with open(blocker, "wb") as fh: + fh.write(b"x") + result = sync_shared_home_mcp(blocker, self.source) + self.assertEqual(result["copied"], []) + self.assertIsNotNone(result["error"]) + + def test_one_unwritable_file_does_not_block_others(self): + _write_source(self.source, self.contents) + os.makedirs(self.target) + bad = SHARED_HOME_MCP_FILES[0] + os.makedirs(os.path.join(self.target, bad)) + result = sync_shared_home_mcp(self.target, self.source) + self.assertEqual(len(result["copied"]), + len(SHARED_HOME_MCP_FILES) - 1) + self.assertIn(bad, result["error"]) + for name in SHARED_HOME_MCP_FILES[1:]: + self.assertTrue(os.path.isfile(os.path.join(self.target, name))) + + def test_no_shared_home_does_not_crash(self): + result = sync_shared_home_mcp(None, self.source) + self.assertIsNotNone(result["error"]) + self.assertEqual(result["copied"], []) + + +@unittest.skipUnless(_SERVER_OK, "cloud.server import unavailable") +class SharedHomeMcpServerTest(unittest.TestCase): + def test_local_worker_home_wins(self): + import cloud.server as server_mod + with mock.patch.object(server_mod, "_local_worker_codex_home", + return_value="/tmp/fake-shared"): + self.assertEqual(_resolve_shared_home_mcp(object()), + "/tmp/fake-shared") + + def test_tenants_root_parent_is_fallback(self): + import cloud.server as server_mod + + class _FakeServer: + tenants_root = "/tmp/fake-root/tenants" + + with mock.patch.object(server_mod, "_local_worker_codex_home", + return_value=None): + self.assertEqual(_resolve_shared_home_mcp(_FakeServer()), + "/tmp/fake-root/codex-home") + + def test_main_calls_sync_and_survives_sync_error(self): + import cloud.server as server_mod + + class _FakeServer: + rate_limiter = None + server_address = ("127.0.0.1", 0) + + def serve_forever(self): + raise KeyboardInterrupt + + calls = {} + fake = _FakeServer() + + def _fake_sync(target): + calls["target"] = target + return {"copied": [], "refreshed": [], "unchanged": [], + "missing": [], "error": "boom (test)"} + + with mock.patch.object(server_mod, "create_server", + return_value=fake), \ + mock.patch.object(server_mod, "_resolve_shared_home_mcp", + return_value="/tmp/fake-shared"), \ + mock.patch.object(server_mod, "_sync_shared_home_mcp", + side_effect=_fake_sync), \ + mock.patch.object(server_mod, "start_rate_limit_sweeper"): + # No exception: sync error is warning-only, and the + # KeyboardInterrupt from serve_forever ends main(). + _server_main(["--db", "/tmp/fake.db", "--port", "0"]) + self.assertEqual(calls.get("target"), "/tmp/fake-shared") + + +if __name__ == "__main__": + unittest.main()