feat: project directory tree + first-round snapshot

- cloud/tree.py: structured worktree tree (dirs first, ignore
  .git/node_modules, symlinks not followed, entry/depth
  truncation flags, text render for injection)
- GET /tasks/{id}/tree with /files auth + worktree resolution,
  max_depth 1..5, max_entries 1..500, escape 400, missing 404,
  cloud/cross-host 409
- MCP get_task_tree translating to the new endpoint
- worker first-round injection of a repo-checkout top-level
  snapshot (2 layers, 120 entries) after repo resolution,
  labelled as checkout snapshot not worktree; skipped for
  coordinator/resume/fork, silent when no host source
- docs/project-tree.md; tests for tree, endpoint, injection,
  MCP translation
This commit is contained in:
host committed 2026-10-30 14:46:11 +08:00
1 parent 85f280e5e7
commit ce7fb3afd6
9 files changed
+865

No files matched your search

+20
View File
@@ -21,6 +21,7 @@ contributor 等权限校验,403 原样透传。
工具(不包管理端):
health、list_projects、list_tasks、get_task、create_task、
cancel_task、get_task_cost、get_task_trace、get_task_files、
get_task_tree、
get_task_result_full、retry_task、merge_task。
create_task 只传 project_id + prompt + 可选 executor/model/
effort/approval/priority,其余字段走服务端缺省;cancel_task
@@ -82,6 +83,11 @@ TOOLS = [
_tool("get_task_files", "列/读任务 worktree 里的文件;不传 path 列顶层,传相对路径读文件或列目录。",
{"task_id": {"type": "string"},
"path": {"type": "string"}}, ["task_id"]),
_tool("get_task_tree", "取任务 worktree 的结构化目录树(只含名、类型、大小,不读文件内容);不传 path 列 worktree 根,传相对路径列该子目录;max_depth 1..5 缺省 3,max_entries 1..500 缺省 200,截断时 truncated=True。",
{"task_id": {"type": "string"},
"path": {"type": "string"},
"max_depth": {"type": "integer"},
"max_entries": {"type": "integer"}}, ["task_id"]),
_tool("get_task_result_full", "取单个任务的完整结果文本(result_text 只是摘要)。",
{"task_id": {"type": "string"}}, ["task_id"]),
_tool("retry_task", "重试一个失败/中断的任务(服务端状态机校验)。",
@@ -201,6 +207,20 @@ class MCPServer:
path += "?" + urllib.parse.urlencode(
{"path": args["path"]})
return self._wrap(*self.transport("GET", path))
if name == "get_task_tree":
if not args.get("task_id"):
return True, {"error": "task_id required"}
query = {}
if args.get("path"):
query["path"] = args["path"]
if args.get("max_depth") is not None:
query["max_depth"] = args["max_depth"]
if args.get("max_entries") is not None:
query["max_entries"] = args["max_entries"]
path = "/tasks/" + args["task_id"] + "/tree"
if query:
path += "?" + urllib.parse.urlencode(query)
return self._wrap(*self.transport("GET", path))
if name == "get_task_result_full":
if not args.get("task_id"):
return True, {"error": "task_id required"}
+60
View File
@@ -74,6 +74,9 @@ from cloud.peer_messages import ensure_followups_table as _peer_ensure_followups
from cloud.peer_messages import peer_message_prefix as _peer_prefix
from cloud.peer_messages import row_get as _row_get
from cloud.marker_tools import parse_spawn_defaults as _parse_spawn_defaults
from cloud.tree import DEFAULT_MAX_DEPTH as _TREE_DEFAULT_DEPTH
from cloud.tree import DEFAULT_MAX_ENTRIES as _TREE_DEFAULT_ENTRIES
from cloud.tree import build_tree as _build_tree
from cloud.automations import cancel_automation as _auto_cancel
from cloud.automations import create_automation as _auto_create
from cloud.automations import get_automation as _auto_get
@@ -2316,6 +2319,8 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
self._task_timeline(user_id, rest[: -len("/timeline")])
elif rest.endswith("/files") and method == "GET":
self._task_files(user_id, rest[: -len("/files")], query)
elif rest.endswith("/tree") and method == "GET":
self._task_tree(user_id, rest[: -len("/tree")], query)
elif rest.endswith("/browser/state") and method == "GET":
self._browser_state(user_id,
rest[: -len("/browser/state")])
@@ -5214,6 +5219,61 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
self._send_json(200, {"task_id": task_id, "path": rel,
"size": size, "content": content})
def _task_tree(self, user_id, task_id, query):
"""GET /tasks/{id}/tree:结构化目录树(不读文件内容)。
与 /files 同款权限与 worktree 解析:viewer+ 可读,
cloud 任务与跨机 worktree 一律 409,解析不到时 404。
?path=<相对路径>:不传列 worktree 根;传目录列该目录。
?max_depth:1..5,缺省 3;?max_entries:1..500,缺省
200。非法整数一律 400,不静默回落缺省。路径逃逸
400,不存在或非目录 404。条数到顶时 truncated=True,
被 max_depth 截断的目录 children=[] 且
children_truncated=True,空目录则是 children=[] 且
children_truncated=False,两者可区分。
"""
task, err = self._thread_task(user_id, task_id, "read")
if err is not None:
self._send_json(err[0], {"error": err[1]})
return
worktree, werr = _task_worktree_for_files(self.server, task)
if werr is not None:
self._send_json(werr[0], {"error": werr[1]})
return
try:
max_depth = int((query.get("max_depth")
or [str(_TREE_DEFAULT_DEPTH)])[0])
max_entries = int((query.get("max_entries")
or [str(_TREE_DEFAULT_ENTRIES)])[0])
except (TypeError, ValueError):
self._send_json(400, {"error":
"max_depth/max_entries must be integers"})
return
if not 1 <= max_depth <= 5 or not 1 <= max_entries <= 500:
self._send_json(400, {"error":
"max_depth must be 1..5 and "
"max_entries must be 1..500"})
return
rel = (query.get("path") or [""])[0]
try:
tree = _build_tree(worktree, path=rel,
max_depth=max_depth,
max_entries=max_entries)
except ValueError:
self._send_json(400, {"error": "path escapes task worktree"})
return
except FileNotFoundError:
self._send_json(404, {"error": "no such path"})
return
except NotADirectoryError:
self._send_json(404, {"error": "path is not a directory"})
return
tree = dict(tree)
tree["task_id"] = task_id
tree["max_depth"] = max_depth
tree["max_entries"] = max_entries
self._send_json(200, tree)
def _browser_worktree_for(self, user_id, task_id):
"""Auth + worktree resolution shared by browser endpoints.
+24
View File
@@ -43,6 +43,10 @@ class FakeTransport:
return 200, {"files": [{"path": "a.py"}]}
if path == "/tasks/t1/files?path=a.py":
return 200, {"content": "print(1)"}
if path == "/tasks/t1/tree":
return 200, {"task_id": "t1", "entries": [{"name": "a.py"}]}
if path == "/tasks/t1/tree?path=sub&max_depth=2":
return 200, {"task_id": "t1", "path": "sub", "entries": []}
if path == "/tasks/t1/result-full":
return 200, {"result_text_full": "full-result"}
if path == "/tasks/t1/retry":
@@ -210,6 +214,26 @@ class MCPServerTestCase(unittest.TestCase):
"create_project_memory", {"project_id": "p1", "kind": "k"})
self.assertTrue(is_error)
def test_task_tree_tool_translate_to_rest(self):
is_error, payload = self.server.call_tool(
"get_task_tree", {"task_id": "t1"})
self.assertFalse(is_error)
self.assertEqual(self.transport.calls[-1][:2],
("GET", "/tasks/t1/tree"))
self.assertIn("get_task_tree",
[t["name"] for t in
self.server.handle(
{"jsonrpc": "2.0", "id": 9,
"method": "tools/list"})["result"]["tools"]])
is_error, _ = self.server.call_tool(
"get_task_tree",
{"task_id": "t1", "path": "sub", "max_depth": 2})
self.assertFalse(is_error)
self.assertEqual(self.transport.calls[-1][:2],
("GET", "/tasks/t1/tree?path=sub&max_depth=2"))
is_error, _ = self.server.call_tool("get_task_tree", {})
self.assertTrue(is_error)
def test_missing_arg_is_error_not_rest_call(self):
before = len(self.transport.calls)
is_error, payload = self.server.call_tool("list_tasks", {})
+175
View File
@@ -0,0 +1,175 @@
"""GET /tasks/{id}/tree 端点测试:结构化目录树,不读文件内容。
harness 同 test_server_task_files:任务行直接插库,worktree
用 server.host_worktrees_dir / tenants_root 注入到临时目录。
"""
import http.client
import json
import os
import sqlite3
import sys
import tempfile
import threading
import unittest
from pathlib import Path
from urllib.parse import quote
try:
from cloud.auth import AuthStore
from cloud.server import create_server
except ImportError:
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
from cloud.auth import AuthStore
from cloud.server import create_server
class TaskTreeTestCase(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.db_path = os.path.join(self.tmp.name, "control.db")
self.auth = AuthStore(self.db_path)
self.owner_id, self.owner_key = self.auth.create_user("owner")
self.viewer_id, self.viewer_key = self.auth.create_user("viewer")
self.outsider_id, self.outsider_key = self.auth.create_user("outsider")
self.server = create_server(self.db_path, "127.0.0.1", 0)
self.worktrees = os.path.join(self.tmp.name, "host-worktrees")
os.makedirs(self.worktrees)
self.server.host_worktrees_dir = self.worktrees
self.tenants = os.path.join(self.tmp.name, "tenants")
self.server.tenants_root = self.tenants
self.port = self.server.server_address[1]
self.thread = threading.Thread(
target=self.server.serve_forever, daemon=True)
self.thread.start()
status, self.project = self._req(
"POST", "/projects", {"name": "demo"}, self.owner_key)
self.assertEqual(status, 201)
self.auth.set_member(
self.project["id"], self.owner_id, self.viewer_id, "viewer")
self._task("task-tree-1", executor=None)
wt = os.path.join(self.worktrees, "task-tree-1")
os.makedirs(os.path.join(wt, "sub", "deep"))
Path(wt, "report.md").write_text("# Report\n", encoding="utf-8")
Path(wt, "sub", "page.html").write_text("<h1>hi</h1>\n",
encoding="utf-8")
Path(wt, "sub", "deep", "leaf.txt").write_text("leaf\n",
encoding="utf-8")
os.makedirs(os.path.join(wt, ".git"))
Path(wt, ".git", "config").write_text("x", encoding="utf-8")
os.symlink(os.path.join(self.tmp.name, "outside.txt"),
os.path.join(wt, "escape"))
Path(self.tmp.name, "outside.txt").write_text("secret\n",
encoding="utf-8")
self._task("task-cloud-1", executor="cloud")
self._task("task-nowt-1", executor=None)
def tearDown(self):
self.server.shutdown()
self.server.server_close()
self.thread.join(timeout=5)
try:
self.auth.conn.close()
except Exception:
pass
self.tmp.cleanup()
def _task(self, task_id, executor):
conn = sqlite3.connect(self.db_path)
try:
conn.execute(
"INSERT INTO tasks(id, project_id, prompt, status, executor)"
" VALUES(?, ?, ?, ?, ?)",
(task_id, self.project["id"], "do it", "completed", executor))
conn.commit()
finally:
conn.close()
def _req(self, method, path, body=None, key=None):
conn = http.client.HTTPConnection("127.0.0.1", self.port, timeout=10)
hdrs = {}
payload = None
if body is not None:
payload = json.dumps(body).encode("utf-8")
hdrs["Content-Type"] = "application/json"
if key is not None:
hdrs["X-API-Key"] = key
try:
conn.request(method, path, body=payload, headers=hdrs)
resp = conn.getresponse()
data = resp.read()
finally:
conn.close()
try:
parsed = json.loads(data.decode("utf-8")) if data else None
except ValueError:
parsed = None
return resp.status, parsed
def _tree(self, task_id, query=None, key=None):
url = f"/tasks/{task_id}/tree"
if query:
url += "?" + "&".join(
"%s=%s" % (k, quote(str(v))) for k, v in query.items())
return self._req("GET", url,
key=self.owner_key if key is None else key)
def test_root_tree_nested_no_content_no_git(self):
status, body = self._tree("task-tree-1")
self.assertEqual(status, 200, body)
self.assertEqual(body["task_id"], "task-tree-1")
self.assertFalse(body["truncated"])
names = [(e["name"], e["type"]) for e in body["entries"]]
self.assertIn(("sub", "dir"), names)
self.assertIn(("report.md", "file"), names)
self.assertIn(("escape", "symlink"), names)
self.assertNotIn(".git", [e["name"] for e in body["entries"]])
# 不读文件内容:整棵树里没有文件正文。
self.assertNotIn("# Report", json.dumps(body))
sub = [e for e in body["entries"] if e["name"] == "sub"][0]
deep = [e for e in sub["children"] if e["name"] == "deep"][0]
self.assertEqual(deep["children"][0]["path"], "sub/deep/leaf.txt")
def test_subdir_and_depth_and_entries_cap(self):
status, body = self._tree("task-tree-1", {"path": "sub"})
self.assertEqual(status, 200, body)
self.assertEqual(body["path"], "sub")
status, body = self._tree("task-tree-1", {"max_depth": "1"})
self.assertEqual(status, 200, body)
sub = [e for e in body["entries"] if e["name"] == "sub"][0]
self.assertEqual(sub["children"], [])
self.assertTrue(sub["children_truncated"])
status, body = self._tree("task-tree-1", {"max_entries": "1"})
self.assertEqual(status, 200, body)
self.assertTrue(body["truncated"])
self.assertEqual(len(body["entries"]), 1)
def test_bad_params_escape_missing_file_cloud(self):
for query in ({"max_depth": "x"}, {"max_depth": "0"},
{"max_depth": "6"}, {"max_entries": "0"},
{"max_entries": "501"}):
status, _ = self._tree("task-tree-1", query)
self.assertEqual(status, 400, query)
for path in ("../outside.txt", "/etc/passwd", "escape"):
status, _ = self._tree("task-tree-1", {"path": path})
self.assertEqual(status, 400, path)
status, _ = self._tree("task-tree-1", {"path": "nope"})
self.assertEqual(status, 404)
status, _ = self._tree("task-tree-1", {"path": "report.md"})
self.assertEqual(status, 404)
status, _ = self._tree("task-nowt-1")
self.assertEqual(status, 404)
status, _ = self._tree("task-cloud-1")
self.assertEqual(status, 409)
def test_auth(self):
status, _ = self._req("GET", "/tasks/task-tree-1/tree")
self.assertEqual(status, 401)
status, _ = self._tree("task-tree-1", key=self.outsider_key)
self.assertEqual(status, 403)
status, _ = self._tree("task-tree-1", key=self.viewer_key)
self.assertEqual(status, 200)
if __name__ == "__main__":
unittest.main()
+111
View File
@@ -0,0 +1,111 @@
"""cloud/tree.py 单测:结构、忽略、符号链接、截断、渲染。
不联网、不跑模型;只建临时目录。
"""
import os
import sys
import tempfile
import unittest
from pathlib import Path
try:
from cloud.tree import build_tree, render_tree_text
except ImportError:
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
from cloud.tree import build_tree, render_tree_text
class TreeTestCase(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.root = Path(self.tmp.name)
(self.root / "pkg").mkdir()
(self.root / "pkg" / "deep").mkdir()
(self.root / "pkg" / "mod.py").write_text("x", encoding="utf-8")
(self.root / "pkg" / "deep" / "leaf.txt").write_text("y",
encoding="utf-8")
(self.root / "README.md").write_text("# hi\n", encoding="utf-8")
(self.root / ".git").mkdir()
(self.root / ".git" / "config").write_text("x", encoding="utf-8")
(self.root / "node_modules").mkdir()
(self.root / "node_modules" / "pkg").mkdir()
os.symlink("/etc/passwd", self.root / "esc")
def tearDown(self):
self.tmp.cleanup()
def _names(self, tree):
return [(e["name"], e["type"]) for e in tree["entries"]]
def test_root_structure_dirs_first_and_ignore(self):
tree = build_tree(str(self.root))
# 目录在前、同类按名:pkg(目录)在文件之前;.git 与
# node_modules 一律不出现;符号链接列出但不跟进。
self.assertEqual(self._names(tree),
[("pkg", "dir"), ("esc", "symlink"),
("README.md", "file")])
self.assertFalse(tree["truncated"])
pkg = tree["entries"][0]
self.assertEqual([c["name"] for c in pkg["children"]],
["deep", "mod.py"])
self.assertEqual(pkg["children"][0]["children"][0]["name"],
"leaf.txt")
def test_max_depth_cuts_children_with_flag(self):
tree = build_tree(str(self.root), max_depth=1)
pkg = tree["entries"][0]
self.assertEqual(pkg["children"], [])
self.assertTrue(pkg["children_truncated"])
tree2 = build_tree(str(self.root), max_depth=2)
deep = tree2["entries"][0]["children"][0]
self.assertEqual(deep["children"], [])
self.assertTrue(deep["children_truncated"])
def test_max_entries_truncates_deterministically(self):
def total(entries):
return sum(1 + total(e.get("children") or [])
for e in entries)
tree = build_tree(str(self.root), max_entries=2)
self.assertTrue(tree["truncated"])
# 条数按整棵树计(含子层):pkg + deep = 2 条即停,
# pkg 的 children 因此半截且 children_truncated=True。
self.assertEqual(total(tree["entries"]), 2)
self.assertTrue(tree["entries"][0]["children_truncated"])
# 再次跑同一输入,结果必须相同(截断不可随机)。
again = build_tree(str(self.root), max_entries=2)
self.assertEqual(tree, again)
def test_subdir_and_size(self):
tree = build_tree(str(self.root), path="pkg")
self.assertEqual(tree["path"], "pkg")
mod = [e for e in tree["entries"] if e["name"] == "mod.py"][0]
self.assertEqual(mod["path"], "pkg/mod.py")
self.assertEqual(mod["size"], 1)
def test_escape_file_and_missing(self):
with self.assertRaises(ValueError):
build_tree(str(self.root), path="../outside")
with self.assertRaises(ValueError):
build_tree(str(self.root), path="/etc/passwd")
with self.assertRaises(FileNotFoundError):
build_tree(str(self.root), path="nope")
with self.assertRaises(NotADirectoryError):
build_tree(str(self.root), path="README.md")
# 指向 worktree 外的符号链接作 path 时同样逃逸。
with self.assertRaises(ValueError):
build_tree(str(self.root), path="esc")
def test_render_text(self):
tree = build_tree(str(self.root), max_depth=2)
text = render_tree_text(tree)
self.assertIn("pkg/", text)
self.assertIn("mod.py", text)
self.assertIn("esc@", text)
self.assertNotIn(".git", text)
self.assertIn("…", text) # max_depth=2 时 deep 的子层被截断
if __name__ == "__main__":
unittest.main()
+110
View File
@@ -0,0 +1,110 @@
"""worker 首轮目录树注入单测:只改内存副本,跳过规则与来源。
不跑真实模型、不派发:直接调 _apply_tree_injection,Worker
用 test_worker 同款最小构造(db 在临时目录、假 host.py)。
"""
import os
import sys
import tempfile
import unittest
from pathlib import Path
try:
from cloud.server import init_db
from cloud.worker import Worker
except ImportError:
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
from cloud.server import init_db
from cloud.worker import Worker
def _completed_runner(cmd):
from types import SimpleNamespace
return SimpleNamespace(returncode=0, stdout="ok", stderr="")
class TreeInjectionTestCase(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.db_path = os.path.join(self.tmp.name, "control.db")
init_db(self.db_path)
self.repo = os.path.join(self.tmp.name, "repo")
os.makedirs(os.path.join(self.repo, "pkg"))
Path(self.repo, "pkg", "mod.py").write_text("x", encoding="utf-8")
Path(self.repo, "README.md").write_text("hi", encoding="utf-8")
os.makedirs(os.path.join(self.repo, ".git"))
self.worker = Worker(
self.db_path, "/fake/host.py",
runner=_completed_runner,
env={"XCM_API_KEY": "test-placeholder"})
def tearDown(self):
self.tmp.cleanup()
def _task(self, **over):
task = {"id": "t1", "project_id": "p1", "prompt": "做一件事",
"repo": self.repo}
task.update(over)
return task
def test_injects_snapshot_and_keeps_prompt(self):
out = self.worker._apply_tree_injection(self._task())
self.assertTrue(out["_tree_injected"])
self.assertTrue(out["prompt"].endswith("做一件事"))
self.assertIn("[项目目录结构:仓库检出顶层快照", out["prompt"])
self.assertIn("pkg/", out["prompt"])
self.assertIn("mod.py", out["prompt"])
self.assertNotIn(".git", out["prompt"])
# 原任务不被改动:注入只改内存副本。
original = self._task()
self.worker._apply_tree_injection(original)
self.assertEqual(original["prompt"], "做一件事")
self.assertNotIn("_tree_injected", original)
def test_goal_task_updates_goal_too(self):
out = self.worker._apply_tree_injection(
self._task(goal_mode=1, goal="目标"))
self.assertIn("[项目目录结构", out["goal"])
self.assertTrue(out["goal"].endswith("目标"))
def test_skip_coordinator_resume_fork_and_no_source(self):
for over in ({"coordinator_parts": [{"name": "a"}]},
{"_resume": True},
{"_goal_resume_breakpoint": True},
{"forked_from": "parent"}):
out = self.worker._apply_tree_injection(self._task(**over))
self.assertEqual(out["prompt"], "做一件事", over)
self.assertNotIn("_tree_injected", out)
out = self.worker._apply_tree_injection(
self._task(repo="/nonexistent/repo"))
self.assertEqual(out["prompt"], "做一件事")
self.assertNotIn("_tree_injected", out)
def test_tenant_missing_checkout_with_repo_url_never_falls_back(self):
# 租户 + 有 repo_url + 检出缺失时,绝不回落到 test_repo
# 模板快照(那会误导);repo_url 为空时才回落。
from cloud.sandbox import DockerExecutor
test_repo = os.path.join(self.tmp.name, "test-repo")
os.makedirs(test_repo)
Path(test_repo, "sample.py").write_text("x", encoding="utf-8")
self.worker.tenant_isolation = True
self.worker.executor = DockerExecutor.__new__(DockerExecutor)
self.worker.tenants_root = os.path.join(self.tmp.name, "tenants")
self.worker._tenant_dirs = lambda pid: {"test_repo": test_repo}
task = self._task(repo="/var/lib/codex-app-server/repos/p1")
self.worker._project_repo_url = lambda pid: "https://example.invalid/r.git"
self.assertIsNone(self.worker._tree_snapshot_source(task))
self.worker._project_repo_url = lambda pid: None
self.assertEqual(self.worker._tree_snapshot_source(task), test_repo)
def test_empty_repo_no_section(self):
empty = os.path.join(self.tmp.name, "empty")
os.makedirs(empty)
out = self.worker._apply_tree_injection(self._task(repo=empty))
self.assertEqual(out["prompt"], "做一件事")
self.assertNotIn("_tree_injected", out)
if __name__ == "__main__":
unittest.main()
+200
View File
@@ -0,0 +1,200 @@
"""任务 worktree 的结构化目录树(server / MCP / worker 首轮注入共用)。
只做一件事:把一个 worktree(或其子目录)列成结构化树,
不读文件内容、不跟符号链接、不出 worktree。
安全规则与 /files 端点同款:
- 相对路径 only:绝对路径、.. 逃逸、符号链接逃逸一律拒绝;
目录内的符号链接列为 type=symlink、size=0、不跟进、不展开。
- .git 永远不列(即使调用方显式传 ignore 也强制加上);
默认还忽略 node_modules、__pycache__、.venv、venv 等
构建/依赖噪音目录。.gitignore 不解析——解析会随仓库
变化而让同一调用返回不同结构,显式忽略集更可预测。
- 条数封顶:遍历按“目录在前、同类按名排序”的确定顺序,
到达 max_entries 即停并置 truncated=True,绝不静默少列。
返回形状:
{"path": <相对路径,根为 "">, "truncated": bool,
"entries": [{"name", "path", "type": "dir"|"file"|"symlink",
"size": int(文件才有意义,目录/链接为 0),
"children": [...](仅 dir 有;
被 max_depth 截断的目录 children=[] 且
children_truncated=True)}]}
max_depth 从请求根算起:1 只列根的直接条目,2 再列一层
子目录,以此类推;默认 3。render_tree_text 把同一结构
渲染成首轮注入用的纯文本树(只含名、目录斜杠与链接 @,
不含大小,省 token)。
"""
import os
__all__ = [
"DEFAULT_IGNORE",
"DEFAULT_MAX_DEPTH",
"DEFAULT_MAX_ENTRIES",
"INJECTION_MAX_DEPTH",
"INJECTION_MAX_ENTRIES",
"build_tree",
"render_tree_text",
"resolve_tree_path",
]
DEFAULT_IGNORE = frozenset({
".git",
"node_modules",
"__pycache__",
".venv",
"venv",
".tox",
".eggs",
})
DEFAULT_MAX_DEPTH = 3
DEFAULT_MAX_ENTRIES = 200
# 首轮注入比 API 更紧:注入是每个任务都付的 prompt 成本,
# 2 层 + 120 条足够定位, deeper 靠 shell 或 get_task_tree 再查。
INJECTION_MAX_DEPTH = 2
INJECTION_MAX_ENTRIES = 120
def resolve_tree_path(root, rel):
"""worktree 内相对路径的安全解析:逃逸返回 None。
空串返回 root 本身;绝对路径、.. 逃逸、符号链接逃逸
(realpath 后越界)一律 None。目录内指回内部的链接
放行到其真实位置——调用方仍只拿到 worktree 内的内容,
但 build_tree 遇到链接本身时不跟进(见下),所以这条
放行只影响“显式 path 指向内部链接目录”时的根解析。
"""
if not rel:
return root
if os.path.isabs(rel):
return None
base = os.path.realpath(root)
target = os.path.realpath(os.path.join(base, rel))
try:
if os.path.commonpath([base, target]) != base:
return None
except ValueError:
return None
return target
def build_tree(root, path="", max_depth=DEFAULT_MAX_DEPTH,
max_entries=DEFAULT_MAX_ENTRIES, ignore=None):
"""列 root 下 path 子目录的结构化树。
path 为空时列 root 本身。path 逃逸时抛 ValueError;
目标不存在时抛 FileNotFoundError;目标不是目录时抛
NotADirectoryError——调用方按各自的 HTTP 语义映射
(server:逃逸 400、不存在/非目录 404)。
"""
target = resolve_tree_path(root, path)
if target is None:
raise ValueError("path escapes task worktree")
if not os.path.exists(target):
raise FileNotFoundError("no such path: %s" % path)
if not os.path.isdir(target):
raise NotADirectoryError("path is not a directory: %s" % path)
effective_ignore = set(ignore) if ignore is not None \
else set(DEFAULT_IGNORE)
effective_ignore.add(".git")
state = {"count": 0, "truncated": False}
entries = _walk(target, root, path, depth=1,
max_depth=max(1, int(max_depth)),
max_entries=max(1, int(max_entries)),
ignore=effective_ignore, state=state)
return {"path": path, "truncated": state["truncated"],
"entries": entries}
def _walk(dir_path, root, rel_prefix, depth, max_depth, max_entries,
ignore, state):
if depth > max_depth:
return []
try:
names = os.listdir(dir_path)
except OSError:
return []
dirs = []
others = []
for name in names:
if name in ignore:
continue
full = os.path.join(dir_path, name)
if os.path.islink(full):
others.append(name)
elif os.path.isdir(full):
dirs.append(name)
else:
others.append(name)
# 目录在前、同类按名排序:遍历顺序确定,截断结果可复现。
ordered = sorted(dirs, key=str.lower) + sorted(others, key=str.lower)
entries = []
for name in ordered:
if state["count"] >= max_entries:
state["truncated"] = True
break
full = os.path.join(dir_path, name)
rel = (rel_prefix + "/" + name).lstrip("/") \
if rel_prefix else name
# Windows 风格分隔符不进结构:统一用 /。
rel = rel.replace(os.sep, "/")
state["count"] += 1
if os.path.islink(full):
entries.append({"name": name, "path": rel,
"type": "symlink", "size": 0})
elif os.path.isdir(full):
if depth >= max_depth:
entries.append({"name": name, "path": rel,
"type": "dir", "size": 0,
"children": [],
"children_truncated": True})
else:
children = _walk(full, root, rel, depth + 1,
max_depth, max_entries, ignore, state)
entries.append({"name": name, "path": rel,
"type": "dir", "size": 0,
"children": children,
# 条数截断发生在子层时同样要标出来,
# 否则半截 children 会被当成完整目录。
"children_truncated": state["truncated"]})
else:
try:
size = os.path.getsize(full)
except OSError:
size = 0
entries.append({"name": name, "path": rel,
"type": "file", "size": size})
if state["truncated"]:
break
return entries
def render_tree_text(tree):
"""把 build_tree 的结果渲染成纯文本树(首轮注入用)。
只含名与目录斜杠,不含大小;截断时末尾明写“已截断”。
"""
lines = ["."]
_render_entries(tree.get("entries") or [], "", lines)
if tree.get("truncated"):
lines.append("(以上已达条数上限,未列全;深层结构用 shell "
"或 get_task_tree 再查。)")
return "\n".join(lines)
def _render_entries(entries, prefix, lines):
for index, entry in enumerate(entries):
last = index == len(entries) - 1
branch = "└── " if last else "├── "
suffix = "/" if entry.get("type") == "dir" else (
"@" if entry.get("type") == "symlink" else "")
lines.append(prefix + branch + entry["name"] + suffix)
children = entry.get("children") or []
if entry.get("type") == "dir" and children:
extension = " " if last else "│ "
_render_entries(children, prefix + extension, lines)
elif entry.get("type") == "dir" \
and entry.get("children_truncated"):
extension = " " if last else "│ "
lines.append(prefix + extension + "…")
+109
View File
@@ -166,6 +166,10 @@ from cloud.todos import create_todo as _todo_create_db
from cloud.todos import get_todo as _todo_get_db
from cloud.todos import list_todos as _todo_list_db
from cloud.todos import update_todo as _todo_update_db
from cloud.tree import INJECTION_MAX_DEPTH as _TREE_INJ_DEPTH
from cloud.tree import INJECTION_MAX_ENTRIES as _TREE_INJ_ENTRIES
from cloud.tree import build_tree as _build_tree
from cloud.tree import render_tree_text as _render_tree_text
from cloud.marker_tools import new_child_id as _marker_new_id
from cloud.marker_tools import parent_spawn_generation as _marker_generation
from cloud.marker_tools import MAX_DIRECT_CHILDREN as _MARKER_MAX_DIRECT
@@ -1836,6 +1840,104 @@ class Worker:
task["id"])
return new_task
def _tree_snapshot_source(self, task):
"""首轮目录树快照的宿主侧来源目录,或 None。
时序诚实点:认领时任务 worktree 还没建出来(派发时
host.py 才按检出建 worktree),所以这里快照的不是
worktree 本身,而是派发源——即 _apply_project_repo
已解析好的检出,worktree 正是由它建出;注入文本里
明写这一点,附件 seed 与 agent 新建文件不在快照内。
候选顺序(只接受宿主上已存在的目录,容器内路径绝
不当宿主路径用):
1. task["repo"] 本身已是宿主目录(非租户、环境任务);
2. 租户 + repo_url:tenants_root/<safe>/<safe>(与
repos.checkout_dir 同规则,_tenant_dirs 的
repo_checkout);
3. 租户 + repo_url 为空:_tenant_dirs 的 test_repo
(派发源就是该模板副本)。
任一候选都不存在时返回 None,调用方静默跳过——
快照缺失绝不炸掉任务认领,agent 仍可用 shell 自查。
"""
repo = task.get("repo")
if repo and os.path.isdir(repo):
return repo
if not (self.tenant_isolation
and isinstance(self.executor, DockerExecutor)):
return None
try:
safe = self._tenant_safe_name(task.get("project_id"))
except Exception:
return None
checkout = os.path.join(self.tenants_root, safe, safe)
if os.path.isdir(checkout):
return checkout
# 检出缺失但项目确有 repo_url 时绝不回落到 test_repo:
# 那说明检出被删/解析出了问题,贴模板快照会误导;
# 只有 repo_url 为空(仓库门本就走 test_repo 路)时
# 才回落。查不到 repo_url 时仓库门同款走 test_repo,
# 故 None 在这里同样回落,不另作区分。
if self._project_repo_url(task.get("project_id")):
return None
try:
dirs = self._tenant_dirs(task.get("project_id"))
except Exception:
return None
test_repo = dirs.get("test_repo") if isinstance(dirs, dict) \
else None
if test_repo and os.path.isdir(test_repo):
return test_repo
return None
def _apply_tree_injection(self, task):
"""首轮注入项目目录结构:只改内存副本,原 prompt/goal
不落库;失败静默跳过。
跳过:协调员任务、续跑与断点续跑、分支任务
(forked_from:其 worktree 来自分支,不是检出快照,
贴检出树会误导)。goal 任务不跳过——结构是数据不是
目标,与项目注入同款同时改内存里的 goal。注入在
run_one 里 _apply_project_repo 之后调用(检出已解析),
不在认领时的注入块里(那时 task["repo"] 还没定)。
上限:2 层、120 条(见 tree.INJECTION_*),深层用
shell 或任务外 get_task_tree 再查;空目录(无条目)
不贴空章节。
"""
if task.get("coordinator_parts"):
return task
if task.get("_resume") or task.get("_goal_resume_breakpoint"):
return task
if task.get("forked_from"):
return task
try:
source = self._tree_snapshot_source(task)
if not source:
return task
tree = _build_tree(source, max_depth=_TREE_INJ_DEPTH,
max_entries=_TREE_INJ_ENTRIES)
except Exception as exc:
try:
self.logger.warning("tree injection failed: %s", exc)
except Exception:
pass
return task
if not tree.get("entries"):
return task
lines = ["[项目目录结构:仓库检出顶层快照,任务 worktree "
"由此建出;附件 seed 与本轮新建文件不在内,"
"深层结构用 shell 或 get_task_tree 再查]",
_render_tree_text(tree)]
prefix = "\n".join(lines)
new_task = dict(task)
new_task["prompt"] = prefix + "\n\n" + (task.get("prompt") or "")
if task.get("goal_mode") or task.get("verify_command"):
new_task["goal"] = prefix + "\n\n" + (
task.get("goal") or task.get("prompt") or "")
new_task["_tree_injected"] = True
self._audit(task["project_id"], "worker_tree_injected",
task["id"])
return new_task
def process_auto_memories(self, now=None, force=False,
owner_id=None):
"""扫 completed 任务并提取记忆(run_loop 轮次边界调用)。
@@ -5510,6 +5612,13 @@ class Worker:
"stdout": "", "error": repo_error, "events": [],
"cost_record_id": None}
# 目录树注入:仓库门之后、真正执行之前——只有这里
# task["repo"] 已解析,_tree_snapshot_source 才能找到
# 宿主侧检出;认领时的注入块里 repo 还没定,贴不了。
# 只改内存副本,原 prompt/goal 不落库;快照缺失时
# 静默跳过,不影响配额门与后续执行。
task = self._apply_tree_injection(task)
# 配额认领门:排队期间配额被花完(或被调低)时,任务不能
# 再开跑——直接标 interrupted + 配额原因,不记成本(还没花
# 钱),续跑消息不消费,顶上来后重试/续跑可再跑。与 key
+56
View File
@@ -0,0 +1,56 @@
# 项目目录树(完整版)
问题:任务内 agent 本来就能用 shell `ls/find` 看 worktree,
但每次都要自己探路(费轮次/token),大项目噪音大、输出
是纯文本;任务外的 LLM(编辑器、协调方)只能用
`get_task_files` 一层一层列/读,没有“一眼看结构”的入口。
本批补三处,共用 `cloud/tree.py`,不改 codex 本体:
## 1. 结构化目录树工具
- REST:`GET /tasks/{id}/tree?path=&max_depth=&max_entries=`
- 权限与 worktree 解析与 `/files` 同款:viewer+ 可读;
cloud 任务与跨机 worktree 一律 409,解析不到 404。
- `max_depth` 1..5(缺省 3),`max_entries` 1..500
(缺省 200),非法整数 400,不静默回落。
- 只含名、类型(dir/file/symlink)、大小,不读文件内容。
- MCP:`get_task_tree`(cloud/mcp_server.py),参数同上,
直译为上述 REST 调用,服务端错误原样透传为 isError。
## 2. 首轮注入顶层结构
worker 在 `run_one` 里 `_apply_project_repo` 之后调
`_apply_tree_injection`:只改内存副本,原 prompt/goal
不落库;goal 任务同时改内存里的 goal。
时序诚实点(不要把这条改回“worktree 快照”):
认领时任务 worktree 还没建出来(派发时 host.py 才按检出
建 worktree),所以注入贴的是**仓库检出顶层快照**,
worktree 正是由它建出;注入文本里明写这一点,附件
seed 与 agent 新建文件不在快照内。来源只认宿主上已存在
的目录:非租户/环境任务用 `task["repo"]`,租户任务用
`tenants_root/<safe>/<safe>` 检出,repo_url 为空的租户
任务用 `_tenant_dirs` 的 test_repo;都找不到时静默跳过,
绝不炸掉认领。
跳过:协调员任务、续跑与断点续跑、分支任务
(`forked_from`:其 worktree 来自分支,不是检出快照)。
上限:2 层、120 条(`tree.INJECTION_*`),深层用 shell
或 `get_task_tree` 再查;空目录不贴空章节。
## 3. 安全与截断规则(tree.py)
- 相对路径 only:绝对路径、`..` 逃逸、符号链接逃逸
一律拒绝;目录内的符号链接列为 symlink、不跟进。
- `.git` 永远不列;默认还忽略 `node_modules`、
`__pycache__`、`.venv`、`venv` 等。`.gitignore` 不解析。
- 遍历顺序确定(目录在前、同类按名排序),条数到顶
置 `truncated=True`;被 `max_depth` 或条数截断的目录
`children_truncated=True`,与空目录(`children=[]` 且
`children_truncated=False`)可区分。
单测:`cloud/tests/test_tree.py`(结构/忽略/链接/截断/
渲染)、`test_server_task_tree.py`(端点/权限/错误码)、
`test_worker_tree_injection.py`(注入/跳过/内存副本)、
`test_mcp_server.py`(`get_task_tree` 翻译)。