feat: project directory tree + first-round snapshot

- cloud/tree.py: structured worktree tree (dirs first, ignore
  .git/node_modules, symlinks not followed, entry/depth
  truncation flags, text render for injection)
- GET /tasks/{id}/tree with /files auth + worktree resolution,
  max_depth 1..5, max_entries 1..500, escape 400, missing 404,
  cloud/cross-host 409
- MCP get_task_tree translating to the new endpoint
- worker first-round injection of a repo-checkout top-level
  snapshot (2 layers, 120 entries) after repo resolution,
  labelled as checkout snapshot not worktree; skipped for
  coordinator/resume/fork, silent when no host source
- docs/project-tree.md; tests for tree, endpoint, injection,
  MCP translation
This commit is contained in:
host committed 2026-10-30 14:46:11 +08:00
1 parent 85f280e5e7
commit ce7fb3afd6
9 files changed
+865

No files matched your search

+24
View File
@@ -43,6 +43,10 @@ class FakeTransport:
return 200, {"files": [{"path": "a.py"}]}
if path == "/tasks/t1/files?path=a.py":
return 200, {"content": "print(1)"}
if path == "/tasks/t1/tree":
return 200, {"task_id": "t1", "entries": [{"name": "a.py"}]}
if path == "/tasks/t1/tree?path=sub&max_depth=2":
return 200, {"task_id": "t1", "path": "sub", "entries": []}
if path == "/tasks/t1/result-full":
return 200, {"result_text_full": "full-result"}
if path == "/tasks/t1/retry":
@@ -210,6 +214,26 @@ class MCPServerTestCase(unittest.TestCase):
"create_project_memory", {"project_id": "p1", "kind": "k"})
self.assertTrue(is_error)
def test_task_tree_tool_translate_to_rest(self):
is_error, payload = self.server.call_tool(
"get_task_tree", {"task_id": "t1"})
self.assertFalse(is_error)
self.assertEqual(self.transport.calls[-1][:2],
("GET", "/tasks/t1/tree"))
self.assertIn("get_task_tree",
[t["name"] for t in
self.server.handle(
{"jsonrpc": "2.0", "id": 9,
"method": "tools/list"})["result"]["tools"]])
is_error, _ = self.server.call_tool(
"get_task_tree",
{"task_id": "t1", "path": "sub", "max_depth": 2})
self.assertFalse(is_error)
self.assertEqual(self.transport.calls[-1][:2],
("GET", "/tasks/t1/tree?path=sub&max_depth=2"))
is_error, _ = self.server.call_tool("get_task_tree", {})
self.assertTrue(is_error)
def test_missing_arg_is_error_not_rest_call(self):
before = len(self.transport.calls)
is_error, payload = self.server.call_tool("list_tasks", {})
+175
View File
@@ -0,0 +1,175 @@
"""GET /tasks/{id}/tree 端点测试:结构化目录树,不读文件内容。
harness 同 test_server_task_files:任务行直接插库,worktree
用 server.host_worktrees_dir / tenants_root 注入到临时目录。
"""
import http.client
import json
import os
import sqlite3
import sys
import tempfile
import threading
import unittest
from pathlib import Path
from urllib.parse import quote
try:
from cloud.auth import AuthStore
from cloud.server import create_server
except ImportError:
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
from cloud.auth import AuthStore
from cloud.server import create_server
class TaskTreeTestCase(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.db_path = os.path.join(self.tmp.name, "control.db")
self.auth = AuthStore(self.db_path)
self.owner_id, self.owner_key = self.auth.create_user("owner")
self.viewer_id, self.viewer_key = self.auth.create_user("viewer")
self.outsider_id, self.outsider_key = self.auth.create_user("outsider")
self.server = create_server(self.db_path, "127.0.0.1", 0)
self.worktrees = os.path.join(self.tmp.name, "host-worktrees")
os.makedirs(self.worktrees)
self.server.host_worktrees_dir = self.worktrees
self.tenants = os.path.join(self.tmp.name, "tenants")
self.server.tenants_root = self.tenants
self.port = self.server.server_address[1]
self.thread = threading.Thread(
target=self.server.serve_forever, daemon=True)
self.thread.start()
status, self.project = self._req(
"POST", "/projects", {"name": "demo"}, self.owner_key)
self.assertEqual(status, 201)
self.auth.set_member(
self.project["id"], self.owner_id, self.viewer_id, "viewer")
self._task("task-tree-1", executor=None)
wt = os.path.join(self.worktrees, "task-tree-1")
os.makedirs(os.path.join(wt, "sub", "deep"))
Path(wt, "report.md").write_text("# Report\n", encoding="utf-8")
Path(wt, "sub", "page.html").write_text("<h1>hi</h1>\n",
encoding="utf-8")
Path(wt, "sub", "deep", "leaf.txt").write_text("leaf\n",
encoding="utf-8")
os.makedirs(os.path.join(wt, ".git"))
Path(wt, ".git", "config").write_text("x", encoding="utf-8")
os.symlink(os.path.join(self.tmp.name, "outside.txt"),
os.path.join(wt, "escape"))
Path(self.tmp.name, "outside.txt").write_text("secret\n",
encoding="utf-8")
self._task("task-cloud-1", executor="cloud")
self._task("task-nowt-1", executor=None)
def tearDown(self):
self.server.shutdown()
self.server.server_close()
self.thread.join(timeout=5)
try:
self.auth.conn.close()
except Exception:
pass
self.tmp.cleanup()
def _task(self, task_id, executor):
conn = sqlite3.connect(self.db_path)
try:
conn.execute(
"INSERT INTO tasks(id, project_id, prompt, status, executor)"
" VALUES(?, ?, ?, ?, ?)",
(task_id, self.project["id"], "do it", "completed", executor))
conn.commit()
finally:
conn.close()
def _req(self, method, path, body=None, key=None):
conn = http.client.HTTPConnection("127.0.0.1", self.port, timeout=10)
hdrs = {}
payload = None
if body is not None:
payload = json.dumps(body).encode("utf-8")
hdrs["Content-Type"] = "application/json"
if key is not None:
hdrs["X-API-Key"] = key
try:
conn.request(method, path, body=payload, headers=hdrs)
resp = conn.getresponse()
data = resp.read()
finally:
conn.close()
try:
parsed = json.loads(data.decode("utf-8")) if data else None
except ValueError:
parsed = None
return resp.status, parsed
def _tree(self, task_id, query=None, key=None):
url = f"/tasks/{task_id}/tree"
if query:
url += "?" + "&".join(
"%s=%s" % (k, quote(str(v))) for k, v in query.items())
return self._req("GET", url,
key=self.owner_key if key is None else key)
def test_root_tree_nested_no_content_no_git(self):
status, body = self._tree("task-tree-1")
self.assertEqual(status, 200, body)
self.assertEqual(body["task_id"], "task-tree-1")
self.assertFalse(body["truncated"])
names = [(e["name"], e["type"]) for e in body["entries"]]
self.assertIn(("sub", "dir"), names)
self.assertIn(("report.md", "file"), names)
self.assertIn(("escape", "symlink"), names)
self.assertNotIn(".git", [e["name"] for e in body["entries"]])
# 不读文件内容:整棵树里没有文件正文。
self.assertNotIn("# Report", json.dumps(body))
sub = [e for e in body["entries"] if e["name"] == "sub"][0]
deep = [e for e in sub["children"] if e["name"] == "deep"][0]
self.assertEqual(deep["children"][0]["path"], "sub/deep/leaf.txt")
def test_subdir_and_depth_and_entries_cap(self):
status, body = self._tree("task-tree-1", {"path": "sub"})
self.assertEqual(status, 200, body)
self.assertEqual(body["path"], "sub")
status, body = self._tree("task-tree-1", {"max_depth": "1"})
self.assertEqual(status, 200, body)
sub = [e for e in body["entries"] if e["name"] == "sub"][0]
self.assertEqual(sub["children"], [])
self.assertTrue(sub["children_truncated"])
status, body = self._tree("task-tree-1", {"max_entries": "1"})
self.assertEqual(status, 200, body)
self.assertTrue(body["truncated"])
self.assertEqual(len(body["entries"]), 1)
def test_bad_params_escape_missing_file_cloud(self):
for query in ({"max_depth": "x"}, {"max_depth": "0"},
{"max_depth": "6"}, {"max_entries": "0"},
{"max_entries": "501"}):
status, _ = self._tree("task-tree-1", query)
self.assertEqual(status, 400, query)
for path in ("../outside.txt", "/etc/passwd", "escape"):
status, _ = self._tree("task-tree-1", {"path": path})
self.assertEqual(status, 400, path)
status, _ = self._tree("task-tree-1", {"path": "nope"})
self.assertEqual(status, 404)
status, _ = self._tree("task-tree-1", {"path": "report.md"})
self.assertEqual(status, 404)
status, _ = self._tree("task-nowt-1")
self.assertEqual(status, 404)
status, _ = self._tree("task-cloud-1")
self.assertEqual(status, 409)
def test_auth(self):
status, _ = self._req("GET", "/tasks/task-tree-1/tree")
self.assertEqual(status, 401)
status, _ = self._tree("task-tree-1", key=self.outsider_key)
self.assertEqual(status, 403)
status, _ = self._tree("task-tree-1", key=self.viewer_key)
self.assertEqual(status, 200)
if __name__ == "__main__":
unittest.main()
+111
View File
@@ -0,0 +1,111 @@
"""cloud/tree.py 单测:结构、忽略、符号链接、截断、渲染。
不联网、不跑模型;只建临时目录。
"""
import os
import sys
import tempfile
import unittest
from pathlib import Path
try:
from cloud.tree import build_tree, render_tree_text
except ImportError:
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
from cloud.tree import build_tree, render_tree_text
class TreeTestCase(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.root = Path(self.tmp.name)
(self.root / "pkg").mkdir()
(self.root / "pkg" / "deep").mkdir()
(self.root / "pkg" / "mod.py").write_text("x", encoding="utf-8")
(self.root / "pkg" / "deep" / "leaf.txt").write_text("y",
encoding="utf-8")
(self.root / "README.md").write_text("# hi\n", encoding="utf-8")
(self.root / ".git").mkdir()
(self.root / ".git" / "config").write_text("x", encoding="utf-8")
(self.root / "node_modules").mkdir()
(self.root / "node_modules" / "pkg").mkdir()
os.symlink("/etc/passwd", self.root / "esc")
def tearDown(self):
self.tmp.cleanup()
def _names(self, tree):
return [(e["name"], e["type"]) for e in tree["entries"]]
def test_root_structure_dirs_first_and_ignore(self):
tree = build_tree(str(self.root))
# 目录在前、同类按名:pkg(目录)在文件之前;.git 与
# node_modules 一律不出现;符号链接列出但不跟进。
self.assertEqual(self._names(tree),
[("pkg", "dir"), ("esc", "symlink"),
("README.md", "file")])
self.assertFalse(tree["truncated"])
pkg = tree["entries"][0]
self.assertEqual([c["name"] for c in pkg["children"]],
["deep", "mod.py"])
self.assertEqual(pkg["children"][0]["children"][0]["name"],
"leaf.txt")
def test_max_depth_cuts_children_with_flag(self):
tree = build_tree(str(self.root), max_depth=1)
pkg = tree["entries"][0]
self.assertEqual(pkg["children"], [])
self.assertTrue(pkg["children_truncated"])
tree2 = build_tree(str(self.root), max_depth=2)
deep = tree2["entries"][0]["children"][0]
self.assertEqual(deep["children"], [])
self.assertTrue(deep["children_truncated"])
def test_max_entries_truncates_deterministically(self):
def total(entries):
return sum(1 + total(e.get("children") or [])
for e in entries)
tree = build_tree(str(self.root), max_entries=2)
self.assertTrue(tree["truncated"])
# 条数按整棵树计(含子层):pkg + deep = 2 条即停,
# pkg 的 children 因此半截且 children_truncated=True。
self.assertEqual(total(tree["entries"]), 2)
self.assertTrue(tree["entries"][0]["children_truncated"])
# 再次跑同一输入,结果必须相同(截断不可随机)。
again = build_tree(str(self.root), max_entries=2)
self.assertEqual(tree, again)
def test_subdir_and_size(self):
tree = build_tree(str(self.root), path="pkg")
self.assertEqual(tree["path"], "pkg")
mod = [e for e in tree["entries"] if e["name"] == "mod.py"][0]
self.assertEqual(mod["path"], "pkg/mod.py")
self.assertEqual(mod["size"], 1)
def test_escape_file_and_missing(self):
with self.assertRaises(ValueError):
build_tree(str(self.root), path="../outside")
with self.assertRaises(ValueError):
build_tree(str(self.root), path="/etc/passwd")
with self.assertRaises(FileNotFoundError):
build_tree(str(self.root), path="nope")
with self.assertRaises(NotADirectoryError):
build_tree(str(self.root), path="README.md")
# 指向 worktree 外的符号链接作 path 时同样逃逸。
with self.assertRaises(ValueError):
build_tree(str(self.root), path="esc")
def test_render_text(self):
tree = build_tree(str(self.root), max_depth=2)
text = render_tree_text(tree)
self.assertIn("pkg/", text)
self.assertIn("mod.py", text)
self.assertIn("esc@", text)
self.assertNotIn(".git", text)
self.assertIn("…", text) # max_depth=2 时 deep 的子层被截断
if __name__ == "__main__":
unittest.main()
+110
View File
@@ -0,0 +1,110 @@
"""worker 首轮目录树注入单测:只改内存副本,跳过规则与来源。
不跑真实模型、不派发:直接调 _apply_tree_injection,Worker
用 test_worker 同款最小构造(db 在临时目录、假 host.py)。
"""
import os
import sys
import tempfile
import unittest
from pathlib import Path
try:
from cloud.server import init_db
from cloud.worker import Worker
except ImportError:
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
from cloud.server import init_db
from cloud.worker import Worker
def _completed_runner(cmd):
from types import SimpleNamespace
return SimpleNamespace(returncode=0, stdout="ok", stderr="")
class TreeInjectionTestCase(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.db_path = os.path.join(self.tmp.name, "control.db")
init_db(self.db_path)
self.repo = os.path.join(self.tmp.name, "repo")
os.makedirs(os.path.join(self.repo, "pkg"))
Path(self.repo, "pkg", "mod.py").write_text("x", encoding="utf-8")
Path(self.repo, "README.md").write_text("hi", encoding="utf-8")
os.makedirs(os.path.join(self.repo, ".git"))
self.worker = Worker(
self.db_path, "/fake/host.py",
runner=_completed_runner,
env={"XCM_API_KEY": "test-placeholder"})
def tearDown(self):
self.tmp.cleanup()
def _task(self, **over):
task = {"id": "t1", "project_id": "p1", "prompt": "做一件事",
"repo": self.repo}
task.update(over)
return task
def test_injects_snapshot_and_keeps_prompt(self):
out = self.worker._apply_tree_injection(self._task())
self.assertTrue(out["_tree_injected"])
self.assertTrue(out["prompt"].endswith("做一件事"))
self.assertIn("[项目目录结构:仓库检出顶层快照", out["prompt"])
self.assertIn("pkg/", out["prompt"])
self.assertIn("mod.py", out["prompt"])
self.assertNotIn(".git", out["prompt"])
# 原任务不被改动:注入只改内存副本。
original = self._task()
self.worker._apply_tree_injection(original)
self.assertEqual(original["prompt"], "做一件事")
self.assertNotIn("_tree_injected", original)
def test_goal_task_updates_goal_too(self):
out = self.worker._apply_tree_injection(
self._task(goal_mode=1, goal="目标"))
self.assertIn("[项目目录结构", out["goal"])
self.assertTrue(out["goal"].endswith("目标"))
def test_skip_coordinator_resume_fork_and_no_source(self):
for over in ({"coordinator_parts": [{"name": "a"}]},
{"_resume": True},
{"_goal_resume_breakpoint": True},
{"forked_from": "parent"}):
out = self.worker._apply_tree_injection(self._task(**over))
self.assertEqual(out["prompt"], "做一件事", over)
self.assertNotIn("_tree_injected", out)
out = self.worker._apply_tree_injection(
self._task(repo="/nonexistent/repo"))
self.assertEqual(out["prompt"], "做一件事")
self.assertNotIn("_tree_injected", out)
def test_tenant_missing_checkout_with_repo_url_never_falls_back(self):
# 租户 + 有 repo_url + 检出缺失时,绝不回落到 test_repo
# 模板快照(那会误导);repo_url 为空时才回落。
from cloud.sandbox import DockerExecutor
test_repo = os.path.join(self.tmp.name, "test-repo")
os.makedirs(test_repo)
Path(test_repo, "sample.py").write_text("x", encoding="utf-8")
self.worker.tenant_isolation = True
self.worker.executor = DockerExecutor.__new__(DockerExecutor)
self.worker.tenants_root = os.path.join(self.tmp.name, "tenants")
self.worker._tenant_dirs = lambda pid: {"test_repo": test_repo}
task = self._task(repo="/var/lib/codex-app-server/repos/p1")
self.worker._project_repo_url = lambda pid: "https://example.invalid/r.git"
self.assertIsNone(self.worker._tree_snapshot_source(task))
self.worker._project_repo_url = lambda pid: None
self.assertEqual(self.worker._tree_snapshot_source(task), test_repo)
def test_empty_repo_no_section(self):
empty = os.path.join(self.tmp.name, "empty")
os.makedirs(empty)
out = self.worker._apply_tree_injection(self._task(repo=empty))
self.assertEqual(out["prompt"], "做一件事")
self.assertNotIn("_tree_injected", out)
if __name__ == "__main__":
unittest.main()