diff --git a/cloud/mcp_server.py b/cloud/mcp_server.py index ef5d4f0..29fa221 100644 --- a/cloud/mcp_server.py +++ b/cloud/mcp_server.py @@ -21,6 +21,7 @@ contributor 等权限校验,403 原样透传。 工具(不包管理端): health、list_projects、list_tasks、get_task、create_task、 cancel_task、get_task_cost、get_task_trace、get_task_files、 + get_task_tree、 get_task_result_full、retry_task、merge_task。 create_task 只传 project_id + prompt + 可选 executor/model/ effort/approval/priority,其余字段走服务端缺省;cancel_task @@ -82,6 +83,11 @@ TOOLS = [ _tool("get_task_files", "列/读任务 worktree 里的文件;不传 path 列顶层,传相对路径读文件或列目录。", {"task_id": {"type": "string"}, "path": {"type": "string"}}, ["task_id"]), + _tool("get_task_tree", "取任务 worktree 的结构化目录树(只含名、类型、大小,不读文件内容);不传 path 列 worktree 根,传相对路径列该子目录;max_depth 1..5 缺省 3,max_entries 1..500 缺省 200,截断时 truncated=True。", + {"task_id": {"type": "string"}, + "path": {"type": "string"}, + "max_depth": {"type": "integer"}, + "max_entries": {"type": "integer"}}, ["task_id"]), _tool("get_task_result_full", "取单个任务的完整结果文本(result_text 只是摘要)。", {"task_id": {"type": "string"}}, ["task_id"]), _tool("retry_task", "重试一个失败/中断的任务(服务端状态机校验)。", @@ -201,6 +207,20 @@ class MCPServer: path += "?" + urllib.parse.urlencode( {"path": args["path"]}) return self._wrap(*self.transport("GET", path)) + if name == "get_task_tree": + if not args.get("task_id"): + return True, {"error": "task_id required"} + query = {} + if args.get("path"): + query["path"] = args["path"] + if args.get("max_depth") is not None: + query["max_depth"] = args["max_depth"] + if args.get("max_entries") is not None: + query["max_entries"] = args["max_entries"] + path = "/tasks/" + args["task_id"] + "/tree" + if query: + path += "?" + urllib.parse.urlencode(query) + return self._wrap(*self.transport("GET", path)) if name == "get_task_result_full": if not args.get("task_id"): return True, {"error": "task_id required"} diff --git a/cloud/server.py b/cloud/server.py index 9d6089d..f1b5dbe 100644 --- a/cloud/server.py +++ b/cloud/server.py @@ -74,6 +74,9 @@ from cloud.peer_messages import ensure_followups_table as _peer_ensure_followups from cloud.peer_messages import peer_message_prefix as _peer_prefix from cloud.peer_messages import row_get as _row_get from cloud.marker_tools import parse_spawn_defaults as _parse_spawn_defaults +from cloud.tree import DEFAULT_MAX_DEPTH as _TREE_DEFAULT_DEPTH +from cloud.tree import DEFAULT_MAX_ENTRIES as _TREE_DEFAULT_ENTRIES +from cloud.tree import build_tree as _build_tree from cloud.automations import cancel_automation as _auto_cancel from cloud.automations import create_automation as _auto_create from cloud.automations import get_automation as _auto_get @@ -2316,6 +2319,8 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler): self._task_timeline(user_id, rest[: -len("/timeline")]) elif rest.endswith("/files") and method == "GET": self._task_files(user_id, rest[: -len("/files")], query) + elif rest.endswith("/tree") and method == "GET": + self._task_tree(user_id, rest[: -len("/tree")], query) elif rest.endswith("/browser/state") and method == "GET": self._browser_state(user_id, rest[: -len("/browser/state")]) @@ -5214,6 +5219,61 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler): self._send_json(200, {"task_id": task_id, "path": rel, "size": size, "content": content}) + def _task_tree(self, user_id, task_id, query): + """GET /tasks/{id}/tree:结构化目录树(不读文件内容)。 + + 与 /files 同款权限与 worktree 解析:viewer+ 可读, + cloud 任务与跨机 worktree 一律 409,解析不到时 404。 + ?path=<相对路径>:不传列 worktree 根;传目录列该目录。 + ?max_depth:1..5,缺省 3;?max_entries:1..500,缺省 + 200。非法整数一律 400,不静默回落缺省。路径逃逸 + 400,不存在或非目录 404。条数到顶时 truncated=True, + 被 max_depth 截断的目录 children=[] 且 + children_truncated=True,空目录则是 children=[] 且 + children_truncated=False,两者可区分。 + """ + task, err = self._thread_task(user_id, task_id, "read") + if err is not None: + self._send_json(err[0], {"error": err[1]}) + return + worktree, werr = _task_worktree_for_files(self.server, task) + if werr is not None: + self._send_json(werr[0], {"error": werr[1]}) + return + try: + max_depth = int((query.get("max_depth") + or [str(_TREE_DEFAULT_DEPTH)])[0]) + max_entries = int((query.get("max_entries") + or [str(_TREE_DEFAULT_ENTRIES)])[0]) + except (TypeError, ValueError): + self._send_json(400, {"error": + "max_depth/max_entries must be integers"}) + return + if not 1 <= max_depth <= 5 or not 1 <= max_entries <= 500: + self._send_json(400, {"error": + "max_depth must be 1..5 and " + "max_entries must be 1..500"}) + return + rel = (query.get("path") or [""])[0] + try: + tree = _build_tree(worktree, path=rel, + max_depth=max_depth, + max_entries=max_entries) + except ValueError: + self._send_json(400, {"error": "path escapes task worktree"}) + return + except FileNotFoundError: + self._send_json(404, {"error": "no such path"}) + return + except NotADirectoryError: + self._send_json(404, {"error": "path is not a directory"}) + return + tree = dict(tree) + tree["task_id"] = task_id + tree["max_depth"] = max_depth + tree["max_entries"] = max_entries + self._send_json(200, tree) + def _browser_worktree_for(self, user_id, task_id): """Auth + worktree resolution shared by browser endpoints. diff --git a/cloud/tests/test_mcp_server.py b/cloud/tests/test_mcp_server.py index d1611d6..f560829 100644 --- a/cloud/tests/test_mcp_server.py +++ b/cloud/tests/test_mcp_server.py @@ -43,6 +43,10 @@ class FakeTransport: return 200, {"files": [{"path": "a.py"}]} if path == "/tasks/t1/files?path=a.py": return 200, {"content": "print(1)"} + if path == "/tasks/t1/tree": + return 200, {"task_id": "t1", "entries": [{"name": "a.py"}]} + if path == "/tasks/t1/tree?path=sub&max_depth=2": + return 200, {"task_id": "t1", "path": "sub", "entries": []} if path == "/tasks/t1/result-full": return 200, {"result_text_full": "full-result"} if path == "/tasks/t1/retry": @@ -210,6 +214,26 @@ class MCPServerTestCase(unittest.TestCase): "create_project_memory", {"project_id": "p1", "kind": "k"}) self.assertTrue(is_error) + def test_task_tree_tool_translate_to_rest(self): + is_error, payload = self.server.call_tool( + "get_task_tree", {"task_id": "t1"}) + self.assertFalse(is_error) + self.assertEqual(self.transport.calls[-1][:2], + ("GET", "/tasks/t1/tree")) + self.assertIn("get_task_tree", + [t["name"] for t in + self.server.handle( + {"jsonrpc": "2.0", "id": 9, + "method": "tools/list"})["result"]["tools"]]) + is_error, _ = self.server.call_tool( + "get_task_tree", + {"task_id": "t1", "path": "sub", "max_depth": 2}) + self.assertFalse(is_error) + self.assertEqual(self.transport.calls[-1][:2], + ("GET", "/tasks/t1/tree?path=sub&max_depth=2")) + is_error, _ = self.server.call_tool("get_task_tree", {}) + self.assertTrue(is_error) + def test_missing_arg_is_error_not_rest_call(self): before = len(self.transport.calls) is_error, payload = self.server.call_tool("list_tasks", {}) diff --git a/cloud/tests/test_server_task_tree.py b/cloud/tests/test_server_task_tree.py new file mode 100644 index 0000000..b7a7643 --- /dev/null +++ b/cloud/tests/test_server_task_tree.py @@ -0,0 +1,175 @@ +"""GET /tasks/{id}/tree 端点测试:结构化目录树,不读文件内容。 + + harness 同 test_server_task_files:任务行直接插库,worktree +用 server.host_worktrees_dir / tenants_root 注入到临时目录。 +""" + +import http.client +import json +import os +import sqlite3 +import sys +import tempfile +import threading +import unittest +from pathlib import Path +from urllib.parse import quote + +try: + from cloud.auth import AuthStore + from cloud.server import create_server +except ImportError: + sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + from cloud.auth import AuthStore + from cloud.server import create_server + + +class TaskTreeTestCase(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.db_path = os.path.join(self.tmp.name, "control.db") + self.auth = AuthStore(self.db_path) + self.owner_id, self.owner_key = self.auth.create_user("owner") + self.viewer_id, self.viewer_key = self.auth.create_user("viewer") + self.outsider_id, self.outsider_key = self.auth.create_user("outsider") + self.server = create_server(self.db_path, "127.0.0.1", 0) + self.worktrees = os.path.join(self.tmp.name, "host-worktrees") + os.makedirs(self.worktrees) + self.server.host_worktrees_dir = self.worktrees + self.tenants = os.path.join(self.tmp.name, "tenants") + self.server.tenants_root = self.tenants + self.port = self.server.server_address[1] + self.thread = threading.Thread( + target=self.server.serve_forever, daemon=True) + self.thread.start() + status, self.project = self._req( + "POST", "/projects", {"name": "demo"}, self.owner_key) + self.assertEqual(status, 201) + self.auth.set_member( + self.project["id"], self.owner_id, self.viewer_id, "viewer") + self._task("task-tree-1", executor=None) + wt = os.path.join(self.worktrees, "task-tree-1") + os.makedirs(os.path.join(wt, "sub", "deep")) + Path(wt, "report.md").write_text("# Report\n", encoding="utf-8") + Path(wt, "sub", "page.html").write_text("

hi

\n", + encoding="utf-8") + Path(wt, "sub", "deep", "leaf.txt").write_text("leaf\n", + encoding="utf-8") + os.makedirs(os.path.join(wt, ".git")) + Path(wt, ".git", "config").write_text("x", encoding="utf-8") + os.symlink(os.path.join(self.tmp.name, "outside.txt"), + os.path.join(wt, "escape")) + Path(self.tmp.name, "outside.txt").write_text("secret\n", + encoding="utf-8") + self._task("task-cloud-1", executor="cloud") + self._task("task-nowt-1", executor=None) + + def tearDown(self): + self.server.shutdown() + self.server.server_close() + self.thread.join(timeout=5) + try: + self.auth.conn.close() + except Exception: + pass + self.tmp.cleanup() + + def _task(self, task_id, executor): + conn = sqlite3.connect(self.db_path) + try: + conn.execute( + "INSERT INTO tasks(id, project_id, prompt, status, executor)" + " VALUES(?, ?, ?, ?, ?)", + (task_id, self.project["id"], "do it", "completed", executor)) + conn.commit() + finally: + conn.close() + + def _req(self, method, path, body=None, key=None): + conn = http.client.HTTPConnection("127.0.0.1", self.port, timeout=10) + hdrs = {} + payload = None + if body is not None: + payload = json.dumps(body).encode("utf-8") + hdrs["Content-Type"] = "application/json" + if key is not None: + hdrs["X-API-Key"] = key + try: + conn.request(method, path, body=payload, headers=hdrs) + resp = conn.getresponse() + data = resp.read() + finally: + conn.close() + try: + parsed = json.loads(data.decode("utf-8")) if data else None + except ValueError: + parsed = None + return resp.status, parsed + + def _tree(self, task_id, query=None, key=None): + url = f"/tasks/{task_id}/tree" + if query: + url += "?" + "&".join( + "%s=%s" % (k, quote(str(v))) for k, v in query.items()) + return self._req("GET", url, + key=self.owner_key if key is None else key) + + def test_root_tree_nested_no_content_no_git(self): + status, body = self._tree("task-tree-1") + self.assertEqual(status, 200, body) + self.assertEqual(body["task_id"], "task-tree-1") + self.assertFalse(body["truncated"]) + names = [(e["name"], e["type"]) for e in body["entries"]] + self.assertIn(("sub", "dir"), names) + self.assertIn(("report.md", "file"), names) + self.assertIn(("escape", "symlink"), names) + self.assertNotIn(".git", [e["name"] for e in body["entries"]]) + # 不读文件内容:整棵树里没有文件正文。 + self.assertNotIn("# Report", json.dumps(body)) + sub = [e for e in body["entries"] if e["name"] == "sub"][0] + deep = [e for e in sub["children"] if e["name"] == "deep"][0] + self.assertEqual(deep["children"][0]["path"], "sub/deep/leaf.txt") + + def test_subdir_and_depth_and_entries_cap(self): + status, body = self._tree("task-tree-1", {"path": "sub"}) + self.assertEqual(status, 200, body) + self.assertEqual(body["path"], "sub") + status, body = self._tree("task-tree-1", {"max_depth": "1"}) + self.assertEqual(status, 200, body) + sub = [e for e in body["entries"] if e["name"] == "sub"][0] + self.assertEqual(sub["children"], []) + self.assertTrue(sub["children_truncated"]) + status, body = self._tree("task-tree-1", {"max_entries": "1"}) + self.assertEqual(status, 200, body) + self.assertTrue(body["truncated"]) + self.assertEqual(len(body["entries"]), 1) + + def test_bad_params_escape_missing_file_cloud(self): + for query in ({"max_depth": "x"}, {"max_depth": "0"}, + {"max_depth": "6"}, {"max_entries": "0"}, + {"max_entries": "501"}): + status, _ = self._tree("task-tree-1", query) + self.assertEqual(status, 400, query) + for path in ("../outside.txt", "/etc/passwd", "escape"): + status, _ = self._tree("task-tree-1", {"path": path}) + self.assertEqual(status, 400, path) + status, _ = self._tree("task-tree-1", {"path": "nope"}) + self.assertEqual(status, 404) + status, _ = self._tree("task-tree-1", {"path": "report.md"}) + self.assertEqual(status, 404) + status, _ = self._tree("task-nowt-1") + self.assertEqual(status, 404) + status, _ = self._tree("task-cloud-1") + self.assertEqual(status, 409) + + def test_auth(self): + status, _ = self._req("GET", "/tasks/task-tree-1/tree") + self.assertEqual(status, 401) + status, _ = self._tree("task-tree-1", key=self.outsider_key) + self.assertEqual(status, 403) + status, _ = self._tree("task-tree-1", key=self.viewer_key) + self.assertEqual(status, 200) + + +if __name__ == "__main__": + unittest.main() diff --git a/cloud/tests/test_tree.py b/cloud/tests/test_tree.py new file mode 100644 index 0000000..d872850 --- /dev/null +++ b/cloud/tests/test_tree.py @@ -0,0 +1,111 @@ +"""cloud/tree.py 单测:结构、忽略、符号链接、截断、渲染。 + +不联网、不跑模型;只建临时目录。 +""" + +import os +import sys +import tempfile +import unittest +from pathlib import Path + +try: + from cloud.tree import build_tree, render_tree_text +except ImportError: + sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + from cloud.tree import build_tree, render_tree_text + + +class TreeTestCase(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.root = Path(self.tmp.name) + (self.root / "pkg").mkdir() + (self.root / "pkg" / "deep").mkdir() + (self.root / "pkg" / "mod.py").write_text("x", encoding="utf-8") + (self.root / "pkg" / "deep" / "leaf.txt").write_text("y", + encoding="utf-8") + (self.root / "README.md").write_text("# hi\n", encoding="utf-8") + (self.root / ".git").mkdir() + (self.root / ".git" / "config").write_text("x", encoding="utf-8") + (self.root / "node_modules").mkdir() + (self.root / "node_modules" / "pkg").mkdir() + os.symlink("/etc/passwd", self.root / "esc") + + def tearDown(self): + self.tmp.cleanup() + + def _names(self, tree): + return [(e["name"], e["type"]) for e in tree["entries"]] + + def test_root_structure_dirs_first_and_ignore(self): + tree = build_tree(str(self.root)) + # 目录在前、同类按名:pkg(目录)在文件之前;.git 与 + # node_modules 一律不出现;符号链接列出但不跟进。 + self.assertEqual(self._names(tree), + [("pkg", "dir"), ("esc", "symlink"), + ("README.md", "file")]) + self.assertFalse(tree["truncated"]) + pkg = tree["entries"][0] + self.assertEqual([c["name"] for c in pkg["children"]], + ["deep", "mod.py"]) + self.assertEqual(pkg["children"][0]["children"][0]["name"], + "leaf.txt") + + def test_max_depth_cuts_children_with_flag(self): + tree = build_tree(str(self.root), max_depth=1) + pkg = tree["entries"][0] + self.assertEqual(pkg["children"], []) + self.assertTrue(pkg["children_truncated"]) + tree2 = build_tree(str(self.root), max_depth=2) + deep = tree2["entries"][0]["children"][0] + self.assertEqual(deep["children"], []) + self.assertTrue(deep["children_truncated"]) + + def test_max_entries_truncates_deterministically(self): + def total(entries): + return sum(1 + total(e.get("children") or []) + for e in entries) + + tree = build_tree(str(self.root), max_entries=2) + self.assertTrue(tree["truncated"]) + # 条数按整棵树计(含子层):pkg + deep = 2 条即停, + # pkg 的 children 因此半截且 children_truncated=True。 + self.assertEqual(total(tree["entries"]), 2) + self.assertTrue(tree["entries"][0]["children_truncated"]) + # 再次跑同一输入,结果必须相同(截断不可随机)。 + again = build_tree(str(self.root), max_entries=2) + self.assertEqual(tree, again) + + def test_subdir_and_size(self): + tree = build_tree(str(self.root), path="pkg") + self.assertEqual(tree["path"], "pkg") + mod = [e for e in tree["entries"] if e["name"] == "mod.py"][0] + self.assertEqual(mod["path"], "pkg/mod.py") + self.assertEqual(mod["size"], 1) + + def test_escape_file_and_missing(self): + with self.assertRaises(ValueError): + build_tree(str(self.root), path="../outside") + with self.assertRaises(ValueError): + build_tree(str(self.root), path="/etc/passwd") + with self.assertRaises(FileNotFoundError): + build_tree(str(self.root), path="nope") + with self.assertRaises(NotADirectoryError): + build_tree(str(self.root), path="README.md") + # 指向 worktree 外的符号链接作 path 时同样逃逸。 + with self.assertRaises(ValueError): + build_tree(str(self.root), path="esc") + + def test_render_text(self): + tree = build_tree(str(self.root), max_depth=2) + text = render_tree_text(tree) + self.assertIn("pkg/", text) + self.assertIn("mod.py", text) + self.assertIn("esc@", text) + self.assertNotIn(".git", text) + self.assertIn("…", text) # max_depth=2 时 deep 的子层被截断 + + +if __name__ == "__main__": + unittest.main() diff --git a/cloud/tests/test_worker_tree_injection.py b/cloud/tests/test_worker_tree_injection.py new file mode 100644 index 0000000..28a5149 --- /dev/null +++ b/cloud/tests/test_worker_tree_injection.py @@ -0,0 +1,110 @@ +"""worker 首轮目录树注入单测:只改内存副本,跳过规则与来源。 + +不跑真实模型、不派发:直接调 _apply_tree_injection,Worker +用 test_worker 同款最小构造(db 在临时目录、假 host.py)。 +""" + +import os +import sys +import tempfile +import unittest +from pathlib import Path + +try: + from cloud.server import init_db + from cloud.worker import Worker +except ImportError: + sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + from cloud.server import init_db + from cloud.worker import Worker + + +def _completed_runner(cmd): + from types import SimpleNamespace + return SimpleNamespace(returncode=0, stdout="ok", stderr="") + + +class TreeInjectionTestCase(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.db_path = os.path.join(self.tmp.name, "control.db") + init_db(self.db_path) + self.repo = os.path.join(self.tmp.name, "repo") + os.makedirs(os.path.join(self.repo, "pkg")) + Path(self.repo, "pkg", "mod.py").write_text("x", encoding="utf-8") + Path(self.repo, "README.md").write_text("hi", encoding="utf-8") + os.makedirs(os.path.join(self.repo, ".git")) + self.worker = Worker( + self.db_path, "/fake/host.py", + runner=_completed_runner, + env={"XCM_API_KEY": "test-placeholder"}) + + def tearDown(self): + self.tmp.cleanup() + + def _task(self, **over): + task = {"id": "t1", "project_id": "p1", "prompt": "做一件事", + "repo": self.repo} + task.update(over) + return task + + def test_injects_snapshot_and_keeps_prompt(self): + out = self.worker._apply_tree_injection(self._task()) + self.assertTrue(out["_tree_injected"]) + self.assertTrue(out["prompt"].endswith("做一件事")) + self.assertIn("[项目目录结构:仓库检出顶层快照", out["prompt"]) + self.assertIn("pkg/", out["prompt"]) + self.assertIn("mod.py", out["prompt"]) + self.assertNotIn(".git", out["prompt"]) + # 原任务不被改动:注入只改内存副本。 + original = self._task() + self.worker._apply_tree_injection(original) + self.assertEqual(original["prompt"], "做一件事") + self.assertNotIn("_tree_injected", original) + + def test_goal_task_updates_goal_too(self): + out = self.worker._apply_tree_injection( + self._task(goal_mode=1, goal="目标")) + self.assertIn("[项目目录结构", out["goal"]) + self.assertTrue(out["goal"].endswith("目标")) + + def test_skip_coordinator_resume_fork_and_no_source(self): + for over in ({"coordinator_parts": [{"name": "a"}]}, + {"_resume": True}, + {"_goal_resume_breakpoint": True}, + {"forked_from": "parent"}): + out = self.worker._apply_tree_injection(self._task(**over)) + self.assertEqual(out["prompt"], "做一件事", over) + self.assertNotIn("_tree_injected", out) + out = self.worker._apply_tree_injection( + self._task(repo="/nonexistent/repo")) + self.assertEqual(out["prompt"], "做一件事") + self.assertNotIn("_tree_injected", out) + + def test_tenant_missing_checkout_with_repo_url_never_falls_back(self): + # 租户 + 有 repo_url + 检出缺失时,绝不回落到 test_repo + # 模板快照(那会误导);repo_url 为空时才回落。 + from cloud.sandbox import DockerExecutor + test_repo = os.path.join(self.tmp.name, "test-repo") + os.makedirs(test_repo) + Path(test_repo, "sample.py").write_text("x", encoding="utf-8") + self.worker.tenant_isolation = True + self.worker.executor = DockerExecutor.__new__(DockerExecutor) + self.worker.tenants_root = os.path.join(self.tmp.name, "tenants") + self.worker._tenant_dirs = lambda pid: {"test_repo": test_repo} + task = self._task(repo="/var/lib/codex-app-server/repos/p1") + self.worker._project_repo_url = lambda pid: "https://example.invalid/r.git" + self.assertIsNone(self.worker._tree_snapshot_source(task)) + self.worker._project_repo_url = lambda pid: None + self.assertEqual(self.worker._tree_snapshot_source(task), test_repo) + + def test_empty_repo_no_section(self): + empty = os.path.join(self.tmp.name, "empty") + os.makedirs(empty) + out = self.worker._apply_tree_injection(self._task(repo=empty)) + self.assertEqual(out["prompt"], "做一件事") + self.assertNotIn("_tree_injected", out) + + +if __name__ == "__main__": + unittest.main() diff --git a/cloud/tree.py b/cloud/tree.py new file mode 100644 index 0000000..981d2a6 --- /dev/null +++ b/cloud/tree.py @@ -0,0 +1,200 @@ +"""任务 worktree 的结构化目录树(server / MCP / worker 首轮注入共用)。 + +只做一件事:把一个 worktree(或其子目录)列成结构化树, +不读文件内容、不跟符号链接、不出 worktree。 + +安全规则与 /files 端点同款: +- 相对路径 only:绝对路径、.. 逃逸、符号链接逃逸一律拒绝; + 目录内的符号链接列为 type=symlink、size=0、不跟进、不展开。 +- .git 永远不列(即使调用方显式传 ignore 也强制加上); + 默认还忽略 node_modules、__pycache__、.venv、venv 等 + 构建/依赖噪音目录。.gitignore 不解析——解析会随仓库 + 变化而让同一调用返回不同结构,显式忽略集更可预测。 +- 条数封顶:遍历按“目录在前、同类按名排序”的确定顺序, + 到达 max_entries 即停并置 truncated=True,绝不静默少列。 + +返回形状: + {"path": <相对路径,根为 "">, "truncated": bool, + "entries": [{"name", "path", "type": "dir"|"file"|"symlink", + "size": int(文件才有意义,目录/链接为 0), + "children": [...](仅 dir 有; + 被 max_depth 截断的目录 children=[] 且 + children_truncated=True)}]} +max_depth 从请求根算起:1 只列根的直接条目,2 再列一层 +子目录,以此类推;默认 3。render_tree_text 把同一结构 +渲染成首轮注入用的纯文本树(只含名、目录斜杠与链接 @, +不含大小,省 token)。 +""" + +import os + +__all__ = [ + "DEFAULT_IGNORE", + "DEFAULT_MAX_DEPTH", + "DEFAULT_MAX_ENTRIES", + "INJECTION_MAX_DEPTH", + "INJECTION_MAX_ENTRIES", + "build_tree", + "render_tree_text", + "resolve_tree_path", +] + +DEFAULT_IGNORE = frozenset({ + ".git", + "node_modules", + "__pycache__", + ".venv", + "venv", + ".tox", + ".eggs", +}) +DEFAULT_MAX_DEPTH = 3 +DEFAULT_MAX_ENTRIES = 200 +# 首轮注入比 API 更紧:注入是每个任务都付的 prompt 成本, +# 2 层 + 120 条足够定位, deeper 靠 shell 或 get_task_tree 再查。 +INJECTION_MAX_DEPTH = 2 +INJECTION_MAX_ENTRIES = 120 + + +def resolve_tree_path(root, rel): + """worktree 内相对路径的安全解析:逃逸返回 None。 + + 空串返回 root 本身;绝对路径、.. 逃逸、符号链接逃逸 + (realpath 后越界)一律 None。目录内指回内部的链接 + 放行到其真实位置——调用方仍只拿到 worktree 内的内容, + 但 build_tree 遇到链接本身时不跟进(见下),所以这条 + 放行只影响“显式 path 指向内部链接目录”时的根解析。 + """ + if not rel: + return root + if os.path.isabs(rel): + return None + base = os.path.realpath(root) + target = os.path.realpath(os.path.join(base, rel)) + try: + if os.path.commonpath([base, target]) != base: + return None + except ValueError: + return None + return target + + +def build_tree(root, path="", max_depth=DEFAULT_MAX_DEPTH, + max_entries=DEFAULT_MAX_ENTRIES, ignore=None): + """列 root 下 path 子目录的结构化树。 + + path 为空时列 root 本身。path 逃逸时抛 ValueError; + 目标不存在时抛 FileNotFoundError;目标不是目录时抛 + NotADirectoryError——调用方按各自的 HTTP 语义映射 + (server:逃逸 400、不存在/非目录 404)。 + """ + target = resolve_tree_path(root, path) + if target is None: + raise ValueError("path escapes task worktree") + if not os.path.exists(target): + raise FileNotFoundError("no such path: %s" % path) + if not os.path.isdir(target): + raise NotADirectoryError("path is not a directory: %s" % path) + effective_ignore = set(ignore) if ignore is not None \ + else set(DEFAULT_IGNORE) + effective_ignore.add(".git") + state = {"count": 0, "truncated": False} + entries = _walk(target, root, path, depth=1, + max_depth=max(1, int(max_depth)), + max_entries=max(1, int(max_entries)), + ignore=effective_ignore, state=state) + return {"path": path, "truncated": state["truncated"], + "entries": entries} + + +def _walk(dir_path, root, rel_prefix, depth, max_depth, max_entries, + ignore, state): + if depth > max_depth: + return [] + try: + names = os.listdir(dir_path) + except OSError: + return [] + dirs = [] + others = [] + for name in names: + if name in ignore: + continue + full = os.path.join(dir_path, name) + if os.path.islink(full): + others.append(name) + elif os.path.isdir(full): + dirs.append(name) + else: + others.append(name) + # 目录在前、同类按名排序:遍历顺序确定,截断结果可复现。 + ordered = sorted(dirs, key=str.lower) + sorted(others, key=str.lower) + entries = [] + for name in ordered: + if state["count"] >= max_entries: + state["truncated"] = True + break + full = os.path.join(dir_path, name) + rel = (rel_prefix + "/" + name).lstrip("/") \ + if rel_prefix else name + # Windows 风格分隔符不进结构:统一用 /。 + rel = rel.replace(os.sep, "/") + state["count"] += 1 + if os.path.islink(full): + entries.append({"name": name, "path": rel, + "type": "symlink", "size": 0}) + elif os.path.isdir(full): + if depth >= max_depth: + entries.append({"name": name, "path": rel, + "type": "dir", "size": 0, + "children": [], + "children_truncated": True}) + else: + children = _walk(full, root, rel, depth + 1, + max_depth, max_entries, ignore, state) + entries.append({"name": name, "path": rel, + "type": "dir", "size": 0, + "children": children, + # 条数截断发生在子层时同样要标出来, + # 否则半截 children 会被当成完整目录。 + "children_truncated": state["truncated"]}) + else: + try: + size = os.path.getsize(full) + except OSError: + size = 0 + entries.append({"name": name, "path": rel, + "type": "file", "size": size}) + if state["truncated"]: + break + return entries + + +def render_tree_text(tree): + """把 build_tree 的结果渲染成纯文本树(首轮注入用)。 + + 只含名与目录斜杠,不含大小;截断时末尾明写“已截断”。 + """ + lines = ["."] + _render_entries(tree.get("entries") or [], "", lines) + if tree.get("truncated"): + lines.append("(以上已达条数上限,未列全;深层结构用 shell " + "或 get_task_tree 再查。)") + return "\n".join(lines) + + +def _render_entries(entries, prefix, lines): + for index, entry in enumerate(entries): + last = index == len(entries) - 1 + branch = "└── " if last else "├── " + suffix = "/" if entry.get("type") == "dir" else ( + "@" if entry.get("type") == "symlink" else "") + lines.append(prefix + branch + entry["name"] + suffix) + children = entry.get("children") or [] + if entry.get("type") == "dir" and children: + extension = " " if last else "│ " + _render_entries(children, prefix + extension, lines) + elif entry.get("type") == "dir" \ + and entry.get("children_truncated"): + extension = " " if last else "│ " + lines.append(prefix + extension + "…") diff --git a/cloud/worker.py b/cloud/worker.py index 662ac2b..cdb35dd 100644 --- a/cloud/worker.py +++ b/cloud/worker.py @@ -166,6 +166,10 @@ from cloud.todos import create_todo as _todo_create_db from cloud.todos import get_todo as _todo_get_db from cloud.todos import list_todos as _todo_list_db from cloud.todos import update_todo as _todo_update_db +from cloud.tree import INJECTION_MAX_DEPTH as _TREE_INJ_DEPTH +from cloud.tree import INJECTION_MAX_ENTRIES as _TREE_INJ_ENTRIES +from cloud.tree import build_tree as _build_tree +from cloud.tree import render_tree_text as _render_tree_text from cloud.marker_tools import new_child_id as _marker_new_id from cloud.marker_tools import parent_spawn_generation as _marker_generation from cloud.marker_tools import MAX_DIRECT_CHILDREN as _MARKER_MAX_DIRECT @@ -1836,6 +1840,104 @@ class Worker: task["id"]) return new_task + def _tree_snapshot_source(self, task): + """首轮目录树快照的宿主侧来源目录,或 None。 + + 时序诚实点:认领时任务 worktree 还没建出来(派发时 + host.py 才按检出建 worktree),所以这里快照的不是 + worktree 本身,而是派发源——即 _apply_project_repo + 已解析好的检出,worktree 正是由它建出;注入文本里 + 明写这一点,附件 seed 与 agent 新建文件不在快照内。 + 候选顺序(只接受宿主上已存在的目录,容器内路径绝 + 不当宿主路径用): + 1. task["repo"] 本身已是宿主目录(非租户、环境任务); + 2. 租户 + repo_url:tenants_root//(与 + repos.checkout_dir 同规则,_tenant_dirs 的 + repo_checkout); + 3. 租户 + repo_url 为空:_tenant_dirs 的 test_repo + (派发源就是该模板副本)。 + 任一候选都不存在时返回 None,调用方静默跳过—— + 快照缺失绝不炸掉任务认领,agent 仍可用 shell 自查。 + """ + repo = task.get("repo") + if repo and os.path.isdir(repo): + return repo + if not (self.tenant_isolation + and isinstance(self.executor, DockerExecutor)): + return None + try: + safe = self._tenant_safe_name(task.get("project_id")) + except Exception: + return None + checkout = os.path.join(self.tenants_root, safe, safe) + if os.path.isdir(checkout): + return checkout + # 检出缺失但项目确有 repo_url 时绝不回落到 test_repo: + # 那说明检出被删/解析出了问题,贴模板快照会误导; + # 只有 repo_url 为空(仓库门本就走 test_repo 路)时 + # 才回落。查不到 repo_url 时仓库门同款走 test_repo, + # 故 None 在这里同样回落,不另作区分。 + if self._project_repo_url(task.get("project_id")): + return None + try: + dirs = self._tenant_dirs(task.get("project_id")) + except Exception: + return None + test_repo = dirs.get("test_repo") if isinstance(dirs, dict) \ + else None + if test_repo and os.path.isdir(test_repo): + return test_repo + return None + + def _apply_tree_injection(self, task): + """首轮注入项目目录结构:只改内存副本,原 prompt/goal + 不落库;失败静默跳过。 + + 跳过:协调员任务、续跑与断点续跑、分支任务 + (forked_from:其 worktree 来自分支,不是检出快照, + 贴检出树会误导)。goal 任务不跳过——结构是数据不是 + 目标,与项目注入同款同时改内存里的 goal。注入在 + run_one 里 _apply_project_repo 之后调用(检出已解析), + 不在认领时的注入块里(那时 task["repo"] 还没定)。 + 上限:2 层、120 条(见 tree.INJECTION_*),深层用 + shell 或任务外 get_task_tree 再查;空目录(无条目) + 不贴空章节。 + """ + if task.get("coordinator_parts"): + return task + if task.get("_resume") or task.get("_goal_resume_breakpoint"): + return task + if task.get("forked_from"): + return task + try: + source = self._tree_snapshot_source(task) + if not source: + return task + tree = _build_tree(source, max_depth=_TREE_INJ_DEPTH, + max_entries=_TREE_INJ_ENTRIES) + except Exception as exc: + try: + self.logger.warning("tree injection failed: %s", exc) + except Exception: + pass + return task + if not tree.get("entries"): + return task + lines = ["[项目目录结构:仓库检出顶层快照,任务 worktree " + "由此建出;附件 seed 与本轮新建文件不在内," + "深层结构用 shell 或 get_task_tree 再查]", + _render_tree_text(tree)] + prefix = "\n".join(lines) + new_task = dict(task) + new_task["prompt"] = prefix + "\n\n" + (task.get("prompt") or "") + if task.get("goal_mode") or task.get("verify_command"): + new_task["goal"] = prefix + "\n\n" + ( + task.get("goal") or task.get("prompt") or "") + new_task["_tree_injected"] = True + self._audit(task["project_id"], "worker_tree_injected", + task["id"]) + return new_task + def process_auto_memories(self, now=None, force=False, owner_id=None): """扫 completed 任务并提取记忆(run_loop 轮次边界调用)。 @@ -5510,6 +5612,13 @@ class Worker: "stdout": "", "error": repo_error, "events": [], "cost_record_id": None} + # 目录树注入:仓库门之后、真正执行之前——只有这里 + # task["repo"] 已解析,_tree_snapshot_source 才能找到 + # 宿主侧检出;认领时的注入块里 repo 还没定,贴不了。 + # 只改内存副本,原 prompt/goal 不落库;快照缺失时 + # 静默跳过,不影响配额门与后续执行。 + task = self._apply_tree_injection(task) + # 配额认领门:排队期间配额被花完(或被调低)时,任务不能 # 再开跑——直接标 interrupted + 配额原因,不记成本(还没花 # 钱),续跑消息不消费,顶上来后重试/续跑可再跑。与 key diff --git a/docs/project-tree.md b/docs/project-tree.md new file mode 100644 index 0000000..2a03e85 --- /dev/null +++ b/docs/project-tree.md @@ -0,0 +1,56 @@ +# 项目目录树(完整版) + +问题:任务内 agent 本来就能用 shell `ls/find` 看 worktree, +但每次都要自己探路(费轮次/token),大项目噪音大、输出 +是纯文本;任务外的 LLM(编辑器、协调方)只能用 +`get_task_files` 一层一层列/读,没有“一眼看结构”的入口。 + +本批补三处,共用 `cloud/tree.py`,不改 codex 本体: + +## 1. 结构化目录树工具 + +- REST:`GET /tasks/{id}/tree?path=&max_depth=&max_entries=` + - 权限与 worktree 解析与 `/files` 同款:viewer+ 可读; + cloud 任务与跨机 worktree 一律 409,解析不到 404。 + - `max_depth` 1..5(缺省 3),`max_entries` 1..500 + (缺省 200),非法整数 400,不静默回落。 + - 只含名、类型(dir/file/symlink)、大小,不读文件内容。 +- MCP:`get_task_tree`(cloud/mcp_server.py),参数同上, + 直译为上述 REST 调用,服务端错误原样透传为 isError。 + +## 2. 首轮注入顶层结构 + +worker 在 `run_one` 里 `_apply_project_repo` 之后调 +`_apply_tree_injection`:只改内存副本,原 prompt/goal +不落库;goal 任务同时改内存里的 goal。 + +时序诚实点(不要把这条改回“worktree 快照”): +认领时任务 worktree 还没建出来(派发时 host.py 才按检出 +建 worktree),所以注入贴的是**仓库检出顶层快照**, +worktree 正是由它建出;注入文本里明写这一点,附件 +seed 与 agent 新建文件不在快照内。来源只认宿主上已存在 +的目录:非租户/环境任务用 `task["repo"]`,租户任务用 +`tenants_root//` 检出,repo_url 为空的租户 +任务用 `_tenant_dirs` 的 test_repo;都找不到时静默跳过, +绝不炸掉认领。 + +跳过:协调员任务、续跑与断点续跑、分支任务 +(`forked_from`:其 worktree 来自分支,不是检出快照)。 +上限:2 层、120 条(`tree.INJECTION_*`),深层用 shell +或 `get_task_tree` 再查;空目录不贴空章节。 + +## 3. 安全与截断规则(tree.py) + +- 相对路径 only:绝对路径、`..` 逃逸、符号链接逃逸 + 一律拒绝;目录内的符号链接列为 symlink、不跟进。 +- `.git` 永远不列;默认还忽略 `node_modules`、 + `__pycache__`、`.venv`、`venv` 等。`.gitignore` 不解析。 +- 遍历顺序确定(目录在前、同类按名排序),条数到顶 + 置 `truncated=True`;被 `max_depth` 或条数截断的目录 + `children_truncated=True`,与空目录(`children=[]` 且 + `children_truncated=False`)可区分。 + +单测:`cloud/tests/test_tree.py`(结构/忽略/链接/截断/ +渲染)、`test_server_task_tree.py`(端点/权限/错误码)、 +`test_worker_tree_injection.py`(注入/跳过/内存副本)、 +`test_mcp_server.py`(`get_task_tree` 翻译)。