harness2: per-role models, sweep batch, task file assets endpoint

- Role models: run_improvement_via_tasks gains role_models
  (probe/editor/selector/reflector) plus --model-<role> CLI flags;
  missing/empty falls back to shared --model, both empty means
  server default, so default behavior is unchanged.
- Sweep: run_sweep + examples/harness2_sweep.py manifest CLI with
  shared library_root (library restore), per-run failure isolation
  and sweep-report.json summary; preflight gates task creation.
- Deliverable assets: GET /tasks/{id}/files for local-executor
  tasks (safe path resolution, UTF-8 text <=256KB, listing <=200;
  docker/cloud/cross-host 409 with explicit reason, since tenant
  Docker shares one /app/work per project and per-task attribution
  would misattribute); probe rollout deliverable_files from the
  endpoint plus DELIVERABLE FILE reply blocks (endpoint wins on
  collision, bad blocks dropped), surfaced in
  deliverables/<label>/files/ context and full_result summary.
- Docs: harness2.md comparison table updated for all three items.
This commit is contained in:
host committed 2026-10-29 16:32:58 +08:00
1 parent 9e444eca38
commit c20192bce8
8 files changed
+856 -28

No files matched your search

+58 -1
View File
@@ -241,6 +241,56 @@ def parse_harness_files(text: str) -> dict[str, str]:
return out
_DELIVERABLE_FILE_START_RX = re.compile(r"^=== DELIVERABLE FILE:\s*(.+?)\s*===$")
_DELIVERABLE_FILE_END = "=== END DELIVERABLE FILE ==="
_DELIVERABLE_FILE_MAX = 8
_DELIVERABLE_FILE_MAX_CHARS = 65536
def _is_safe_deliverable_path(path: str) -> bool:
if not path or path.startswith("/") or "\x00" in path:
return False
return all(p not in ("", ".", "..") for p in path.split("/"))
def parse_deliverable_files(text: str) -> dict[str, str]:
"""解析探针回复里的产物文件块,返回 {相对路径: 内容}。
与编辑文件块的严格语义不同:这里故意宽容——坏块(未闭合、
非法路径、超量、超长)只丢弃该块、不抛错,因为产物文件是
交付文本之外的 enrichment,绝不能因为一块坏内容毁掉整轮
交付物。超长内容截断并打截断标记,让评审知道它不完整;
超过 8 块时只取前 8 块。同一路径出现两次时后一块覆盖前一块。
"""
out: dict[str, str] = {}
cur_path: str | None = None
cur_lines: list[str] = []
for line in (text or "").splitlines():
m = _DELIVERABLE_FILE_START_RX.match(line.strip())
if m:
if cur_path is not None:
cur_path = None
cur_lines = []
path = m.group(1).strip()
if _is_safe_deliverable_path(path) and len(out) < _DELIVERABLE_FILE_MAX:
cur_path = path
cur_lines = []
continue
if line.strip() == _DELIVERABLE_FILE_END:
if cur_path is not None:
content = "\n".join(cur_lines) + ("\n" if cur_lines else "")
if len(content) > _DELIVERABLE_FILE_MAX_CHARS:
content = (content[:_DELIVERABLE_FILE_MAX_CHARS]
+ "\n...[truncated]")
out[cur_path] = content
cur_path = None
cur_lines = []
continue
if cur_path is not None:
cur_lines.append(line)
return out
MAX_K = 3
STAGE_ATTEMPTS = 2 # 提议被拒后给一次按反馈修正的机会,再拒就回落
@@ -490,6 +540,7 @@ class Rollout:
trajectory: str = ""
cost_usd: float = 0.0
error: str = ""
deliverable_files: dict[str, str] = field(default_factory=dict)
@dataclass
@@ -589,7 +640,12 @@ def build_context(
(dd / f"{r.label}.md").write_text(
r.deliverable_text or "(no deliverable)", encoding="utf-8"
)
lines.append("context/deliverables/ what each rollout produced")
for name, content in (r.deliverable_files or {}).items():
dest = dd / r.label / "files" / name
dest.parent.mkdir(parents=True, exist_ok=True)
dest.write_text(content, encoding="utf-8")
lines.append("context/deliverables/ what each rollout produced"
" (+ <label>/files/ when it produced files)")
write_tree(harness, root / "harness")
lines.append("context/harness/ the harness tree you may edit, and ONLY this")
if library_dir and Path(library_dir).is_dir():
@@ -842,6 +898,7 @@ class ModeResult:
"delivered": r.delivered,
"error": (r.error or "")[:400],
"deliverable_text": (r.deliverable_text or "")[:500],
"deliverable_files": sorted(r.deliverable_files or {}),
}
for lbl, r in self.rollouts.items()
},
+145 -13
View File
@@ -1,6 +1,6 @@
"""Harness² 接线层:把 cloud/harness2.py 的三个协议落到真实 codex 任务上。
只用控制面已有的 API,不新增任何服务端端点:
控制面 API:
POST /uploads(multipart + paths JSON)-> upload_id
POST /tasks(upload_id 绑定 + idempotency_key 幂等)-> 任务排队
GET /tasks/{id} 轮询终态(completed / failed / interrupted /
@@ -8,6 +8,8 @@
GET /tasks/{id}/trace 轨迹(探针的 trajectory 来源)
GET /tasks/{id}/cost 成本(usd_micros / 1e6;无价 token 不计入 USD,
这是现有 cost 模型的边界,不是本层漏算)
GET /tasks/{id}/files 读本地任务 worktree 文件(产物资产端点;
Docker/跨机任务回 409,见下)
POST /tasks/{id}/cancel 超时取消,不留孤儿任务继续烧钱
认证:X-API-Key 请求头;密钥只由调用方持有,本模块不读、不存、
不打印密钥。
@@ -18,12 +20,13 @@
文件会覆盖仓库同名文件——这正是探针要的效果:同一任务在不同
harness 下跑。prompt = 原任务 prompt + 一段交付脚注(见
_PROBE_DELIVERABLE_FOOTER),因为评审只看最终回复:agent 必须
把交付物完整写在最终回复里,不能只说“文件已写好”。这就是本层
最大的适用边界:交付物是最终回复文本的任务;文件型产物(代码
diff、报告文件)目前只能靠最终回复复述/摘要,worktree 快照 API
只给 diff 截断与未跟踪文件列表、不给完整文件内容,跨主机还读
不到——文件产物评审需要新增文件读取端点,另行评审,不在本层
假装已解决。
把交付物完整写在最终回复里,不能只说“文件已写好”。
文件型产物走两层(产物资产):①本地任务经 GET /tasks/{id}/files
读 worktree 新增/改动文件(端点为准);②所有执行器都可用
回复里的 DELIVERABLE FILE 块(agent 自报,端点结果优先)。
边界仍在:租户 Docker 下 /app/work 是整项目共享目录,按任务
归属读文件会错指,端点对 Docker 任务一律 409——这时只有第
②层可用,且内容以 agent 自报为准(可能截断,截断有标记)。
- 编辑(CodexEditor.edit):bundle = context/ 前缀的证据树(task.md /
rollouts / deliverables / harness / library / pool / decisions)。
agent 在 worktree 的 context/harness/ 里改文件,最终回复里用
@@ -65,6 +68,7 @@ import json
import re
import time
import urllib.error
import urllib.parse
import urllib.request
import uuid
from dataclasses import dataclass, field
@@ -77,6 +81,7 @@ from .harness2 import (
Tree,
improve_task,
parse_choice,
parse_deliverable_files,
parse_experience,
parse_harness_files,
parse_no_change,
@@ -185,6 +190,11 @@ write the complete deliverable in your final reply itself — do not only say
"done" or "the file is written". If the task asks for a file, also write the
file in the worktree, but the final reply must still contain the full content
a reviewer needs to judge the work.
If you created deliverable files (a page, a report, an image manifest),
also paste each file's full content in your final reply as a block:
=== DELIVERABLE FILE: <relative path> ===
<complete file content>
=== END DELIVERABLE FILE ===
"""
_EDITOR_OUTPUT_FORMAT = """
@@ -278,6 +288,12 @@ class TaskClient(Protocol):
"""轨迹渲染文本;读不到时返回空串(调用方用最终回复兜底)。"""
...
def get_task_files(self, task_id: str, path: str = "") -> dict:
"""GET /tasks/{id}/files:不传 path 列 worktree 文件,传 path
读单个文件。服务端对 Docker/跨机任务回 409,调用方须把
409 当成“无文件通道”、不得当成探针失败。"""
...
def get_cost_usd(self, task_id: str) -> float:
"""有价成本 USD;读不到时返回 0.0,不炸轮次。"""
...
@@ -452,6 +468,10 @@ class HttpTaskClient:
rendered = "\n".join(lines).strip()
return rendered[:TRACE_MAX_CHARS] if rendered else ""
def get_task_files(self, task_id: str, path: str = "") -> dict:
q = f"?path={urllib.parse.quote(path)}" if path else ""
return self._request("GET", f"/tasks/{task_id}/files{q}")
def get_cost_usd(self, task_id: str) -> float:
try:
obj = self._request("GET", f"/tasks/{task_id}/cost")
@@ -596,6 +616,12 @@ class CodexProbeAdapter:
error=f"{type(exc).__name__}: {exc}"[:400])
state = done.state
delivered = state.status == "completed" and bool(state.result_text.strip())
deliverable_files = self._collect_deliverable_files(done.task_id, files)
# 回复里的产物文件块是补充通道(所有执行器都可用);
# 端点读到的文件为准,冲突时不覆盖端点结果。
for name, content in parse_deliverable_files(
state.result_text or "").items():
deliverable_files.setdefault(name, content)
return Rollout(
label=label, ok=state.status == "completed", delivered=delivered,
deliverable_text=state.result_text,
@@ -603,8 +629,46 @@ class CodexProbeAdapter:
cost_usd=done.cost_usd,
error="" if state.status == "completed" else (
state.error_text or f"task ended as {state.status}")[:400],
deliverable_files=deliverable_files,
)
def _collect_deliverable_files(self, task_id: str,
seed_files: dict[str, bytes]) -> dict[str, str]:
"""经 GET /tasks/{id}/files 读探针 worktree 里新增的文件。
取 seed 里没有的新文件,以及内容被改过的 seed 文件(与
seed 字节比对,不一致才算产物);跳过隐藏文件、最多收
8 个、最多读 16 个;端点不可用(Docker/跨机 409、读错)
时返回空 dict——文件产物是 enrichment,绝不能因此炸轮次。
"""
try:
listing = self.client.get_task_files(task_id)
except Exception:
return {}
out: dict[str, str] = {}
reads = 0
for entry in listing.get("entries") or []:
if not isinstance(entry, dict):
continue
path = entry.get("path") or ""
if not path or path.startswith("."):
continue
if len(out) >= 8 or reads >= 16:
break
try:
data = self.client.get_task_files(task_id, path)
except Exception:
continue
reads += 1
content = data.get("content")
if not isinstance(content, str):
continue
seed = seed_files.get(path)
if seed is not None and seed.decode("utf-8", errors="replace") == content:
continue
out[path] = content
return out
@dataclass
class CodexEditor:
@@ -773,6 +837,7 @@ def run_improvement_via_tasks(
budget_usd: float | None = None,
max_sessions: int | None = None,
model: str | None = None,
role_models: dict[str, str | None] | None = None,
effort: str | None = None,
approval: str | None = None,
environment_id: str | None = None,
@@ -785,22 +850,37 @@ def run_improvement_via_tasks(
library_root 为空时不建经验库(只跑改进、不入库);非空时按
domain 建库。auto_apply 恒为 False——采纳的树只入库待评审,
不自动成为后续任务的 base(见 docs/harness2.md 安全一节)。
分角色模型:role_models 键为 probe / editor / selector /
reflector,非空值覆盖该角色的 model;缺键或空值时回落到
共享 model;两者都空时不传 model(服务端默认模型)。默认
行为与此前完全一致——本参数只提供分流能力,不替用户选
模型(有价模型会产生真实成本,见 docs/harness2.md)。
"""
from .harness2 import Library
roles = role_models or {}
def _role_model(role: str) -> str | None:
return roles.get(role) or model
adapter = CodexProbeAdapter(
client=client, project_id=project_id, task_id=task_id, prompt=prompt,
base=base_harness, run_tag=run_tag,
source_files=source_files or {}, environment_id=environment_id,
executor=executor, model=model, effort=effort, approval=approval,
executor=executor, model=_role_model("probe"), effort=effort,
approval=approval,
no_time_limit=no_time_limit, timeout_seconds=timeout_seconds,
domain=domain)
shared = dict(environment_id=environment_id, executor=executor, model=model,
shared = dict(environment_id=environment_id, executor=executor,
effort=effort, approval=approval, no_time_limit=no_time_limit,
timeout_seconds=timeout_seconds)
editor = CodexEditor(client=client, project_id=project_id, run_tag=run_tag, **shared)
selector = CodexSelector(client=client, project_id=project_id, run_tag=run_tag, **shared)
reflector = CodexReflector(client=client, project_id=project_id, run_tag=run_tag, **shared)
editor = CodexEditor(client=client, project_id=project_id, run_tag=run_tag,
model=_role_model("editor"), **shared)
selector = CodexSelector(client=client, project_id=project_id, run_tag=run_tag,
model=_role_model("selector"), **shared)
reflector = CodexReflector(client=client, project_id=project_id, run_tag=run_tag,
model=_role_model("reflector"), **shared)
library = Library(library_root, domain) if library_root is not None else None
validate_run_tag(run_tag)
return improve_task(
@@ -809,10 +889,62 @@ def run_improvement_via_tasks(
auto_apply=False, budget_usd=budget_usd, max_sessions=max_sessions)
def run_sweep(client, *, project_id, runs, work_root, library_root=None):
"""批量跑多轮改进:共享经验库、单轮失败隔离、汇总报告。
runs:每轮一个 dict,键为 run_improvement_via_tasks 的参数,
run_tag 必填;client / project_id / work_dir / library_root
由本函数统管,出现在 spec 里会被忽略(不报错)。每轮
work_dir = work_root/<run_tag>。library_root 非空时所有轮
共享同一库——已入库的经验会被后续轮读到,即上游 sweep 的
library restore 语义;为空时各轮都不入库(仍汇总)。
单轮抛错只记入报告的 error,不中断后续轮(dead 隔离)。
返回报告 dict:runs 逐轮(ok / chosen_label / deliverable
前 500 字 / cost / library_entry / error)+ totals(轮数、
成功轮数、总成本)。
"""
work_root = Path(work_root)
report_runs = []
total_cost = 0.0
n_ok = 0
for spec in runs:
spec = dict(spec)
run_tag = spec.pop("run_tag", "")
for owned in ("client", "project_id", "work_dir", "library_root"):
spec.pop(owned, None)
entry = {"run_tag": run_tag, "ok": False, "chosen_label": None,
"deliverable_text": "", "total_cost_usd": 0.0,
"library_entry": None, "error": None}
try:
if not run_tag:
raise ValueError("sweep 每轮必须给 run_tag(用作 work_dir 子目录)")
validate_run_tag(run_tag)
result = run_improvement_via_tasks(
client, project_id=project_id, run_tag=run_tag,
work_dir=work_root / run_tag,
library_root=library_root, **spec)
entry["ok"] = bool(result.ok)
entry["chosen_label"] = result.selection.chosen_label
chosen = result.mode_result.rollouts.get(result.selection.chosen_label)
if chosen is not None:
entry["deliverable_text"] = (chosen.deliverable_text or "")[:500]
entry["total_cost_usd"] = result.mode_result.total_cost_usd
entry["library_entry"] = result.library_entry
total_cost += result.mode_result.total_cost_usd
n_ok += 1 if result.ok else 0
except Exception as exc:
entry["error"] = f"{type(exc).__name__}: {exc}"[:400]
report_runs.append(entry)
return {"runs": report_runs,
"totals": {"n_runs": len(report_runs), "n_ok": n_ok,
"total_cost_usd": total_cost}}
__all__ = [
"TERMINAL_STATUSES", "DEFAULT_TASK_TIMEOUT_SECONDS", "DEFAULT_POLL_SECONDS",
"TRACE_MAX_CHARS", "TaskState", "TaskClient", "build_multipart",
"HttpTaskClient", "CompletedTask", "run_task",
"CodexProbeAdapter", "CodexEditor", "CodexSelector", "CodexReflector",
"run_improvement_via_tasks", "validate_run_tag", "preflight_checks",
"run_improvement_via_tasks", "run_sweep", "validate_run_tag", "preflight_checks",
]
+99
View File
@@ -445,6 +445,22 @@ def _thread_worktree_for_task(server, task):
return os.path.join(base, safe)
def _safe_worktree_path(worktree, rel):
"""worktree 内相对路径的安全解析:空串返回 worktree 本身;
绝对路径、.. 逃逸、符号链接逃逸一律返回 None。realpath 后
再校验 commonpath,目录内的符号链接指回内部时放行(读到
的仍是 worktree 内的内容),指出去时拦住。"""
if not rel:
return worktree
if os.path.isabs(rel):
return None
base = os.path.realpath(worktree)
target = os.path.realpath(os.path.join(base, rel))
if os.path.commonpath([base, target]) != base:
return None
return target
def _extract_thread_list(result):
"""从 thread-list 的 CLI 结果里取 thread 列表:标准形态是
ThreadListResponse 的 data;threads 键与 result 包裹是
@@ -2152,6 +2168,8 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
self._list_followups(user_id, rest[: -len("/followups")])
elif rest.endswith("/timeline") and method == "GET":
self._task_timeline(user_id, rest[: -len("/timeline")])
elif rest.endswith("/files") and method == "GET":
self._task_files(user_id, rest[: -len("/files")], query)
elif rest.endswith("/chat-history") and method == "GET":
self._task_chat_history(
user_id, rest[: -len("/chat-history")])
@@ -4698,6 +4716,87 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
for r in cp_rows],
})
def _task_files(self, user_id, task_id, query):
"""GET /tasks/{id}/files:读本地任务 worktree 里的文件。
只做 local 执行器任务(复用 thread 操作的执行器门):
Docker 任务的 worktree 在容器内、且租户 Docker 下 /app/work
是整项目共享目录,按任务归属读文件会错指,一律 409 明
说;cloud 任务与跨机 worktree 同理。executor 为空的老
任务放行(与 thread 门同款),解析不到 worktree 时 404。
?path=<相对路径>:不传列 worktree 顶层(递归、最多 200
条,只含文件路径与大小,不跟符号链接);传目录列该目录
(非递归,含 is_dir);传文件读文本内容(UTF-8、≤256KB,
二进制或超大一律 409/413 明说,不截断冒充全文)。路径
解析一律走 _safe_worktree_path,防 .. 与符号链接逃逸。
读操作不限制任务状态(进行中任务读到的是中间态)。
"""
task, err = self._thread_task(user_id, task_id, "read")
if err is not None:
self._send_json(err[0], {"error": err[1]})
return
guard = self._thread_task_guard(task)
if guard is not None:
self._send_json(guard[0], {"error": guard[1]})
return
worktree = _thread_worktree_for_task(self.server, task)
if not os.path.isdir(worktree):
self._send_json(404, {"error": "no local worktree for this task"})
return
rel = (query.get("path") or [""])[0]
target = _safe_worktree_path(worktree, rel)
if target is None:
self._send_json(400, {"error": "path escapes task worktree"})
return
if os.path.isdir(target):
entries = []
if not rel:
for root, dirs, files in os.walk(target):
dirs[:] = [d for d in dirs
if not os.path.islink(os.path.join(root, d))]
dirs.sort()
for name in sorted(files):
fpath = os.path.join(root, name)
if os.path.islink(fpath):
continue
entries.append({
"path": os.path.relpath(fpath, worktree),
"size": os.path.getsize(fpath),
})
if len(entries) >= 200:
break
if len(entries) >= 200:
break
else:
for name in sorted(os.listdir(target)):
fpath = os.path.join(target, name)
is_link = os.path.islink(fpath)
entries.append({
"path": os.path.relpath(fpath, worktree),
"size": os.path.getsize(fpath)
if os.path.isfile(fpath) and not is_link else 0,
"is_dir": os.path.isdir(fpath) and not is_link,
})
self._send_json(200, {"task_id": task_id, "path": rel,
"entries": entries})
return
if not os.path.isfile(target):
self._send_json(404, {"error": "no such file"})
return
size = os.path.getsize(target)
if size > 256 * 1024:
self._send_json(413, {"error": "file too large for inline read"})
return
try:
with open(target, encoding="utf-8") as fh:
content = fh.read()
except (OSError, UnicodeDecodeError):
self._send_json(409, {"error": "file is not readable UTF-8 text"})
return
self._send_json(200, {"task_id": task_id, "path": rel,
"size": size, "content": content})
def _task_chat_history(self, user_id, task_id):
"""GET /tasks/{id}/chat-history:真正的交替历史,一次性
按时间交错拼好返回,前端不再猜顺序。
+201 -10
View File
@@ -19,26 +19,26 @@ from pathlib import Path
try:
from cloud import harness2_tasks
from cloud.harness2 import (
parse_choice, parse_experience, parse_harness_files, parse_no_change,
parse_notes, parse_why,
parse_choice, parse_deliverable_files, parse_experience,
parse_harness_files, parse_no_change, parse_notes, parse_why,
)
from cloud.harness2_tasks import (
CodexEditor, CodexProbeAdapter, HttpTaskClient, TaskState,
build_multipart, preflight_checks, run_improvement_via_tasks, run_task,
unwrap_result_text, validate_run_tag,
build_multipart, preflight_checks, run_improvement_via_tasks, run_sweep,
run_task, unwrap_result_text, validate_run_tag,
)
from cloud.uploads import parse_multipart, parse_paths_field
except ImportError: # discover -s cloud/tests 时把项目根目录补进 sys.path
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
from cloud import harness2_tasks
from cloud.harness2 import (
parse_choice, parse_experience, parse_harness_files, parse_no_change,
parse_notes, parse_why,
parse_choice, parse_deliverable_files, parse_experience,
parse_harness_files, parse_no_change, parse_notes, parse_why,
)
from cloud.harness2_tasks import (
CodexEditor, CodexProbeAdapter, HttpTaskClient, TaskState,
build_multipart, preflight_checks, run_improvement_via_tasks, run_task,
unwrap_result_text, validate_run_tag,
build_multipart, preflight_checks, run_improvement_via_tasks, run_sweep,
run_task, unwrap_result_text, validate_run_tag,
)
from cloud.uploads import parse_multipart, parse_paths_field
@@ -61,15 +61,22 @@ LEDGER = b"amount\n100.25\n-20.10\n5.00\n"
class FakeTaskClient:
"""进程内控制面:按 prompt 类型分派到确定性假 worker。"""
def __init__(self, editor_mode="fix", selector_mode="signed", envelope=False):
def __init__(self, editor_mode="fix", selector_mode="signed", envelope=False,
fail_tags=(), endpoint_files=None, endpoint_error=False,
probe_extra=""):
self.editor_mode = editor_mode
self.selector_mode = selector_mode
self.envelope = envelope
self.fail_tags = tuple(fail_tags)
self.endpoint_files = dict(endpoint_files or {})
self.endpoint_error = endpoint_error
self.probe_extra = probe_extra
self.uploads = {}
self.tasks = {}
self.by_key = {}
self.created = 0
self.cancelled = []
self.task_models = {}
def upload(self, files):
uid = f"u{len(self.uploads) + 1}"
@@ -78,6 +85,8 @@ class FakeTaskClient:
def create_task(self, *, project_id, prompt, upload_id=None,
idempotency_key=None, **kwargs):
if idempotency_key and any(t in idempotency_key for t in self.fail_tags):
raise RuntimeError("injected task failure")
if idempotency_key and idempotency_key in self.by_key:
return self.by_key[idempotency_key]
tid = f"t{len(self.tasks) + 1}"
@@ -94,6 +103,7 @@ class FakeTaskClient:
})
self.tasks[tid] = TaskState(task_id=tid, status="completed",
result_text=result)
self.task_models[tid] = kwargs.get("model")
if idempotency_key:
self.by_key[idempotency_key] = tid
return tid
@@ -163,11 +173,22 @@ class FakeTaskClient:
[sys.executable, str(root / "scripts" / "summarize.py"),
str(root / "ledger.csv")],
capture_output=True, text=True, timeout=10)
return proc.stdout.strip()
return proc.stdout.strip() + self.probe_extra
def get_task(self, task_id):
return self.tasks[task_id]
def get_task_files(self, task_id, path=""):
if self.endpoint_error:
raise RuntimeError("409 files unavailable")
if not path:
return {"entries": [
{"path": p, "size": len(c.encode("utf-8"))}
for p, c in sorted(self.endpoint_files.items())]}
if path in self.endpoint_files:
return {"content": self.endpoint_files[path]}
raise RuntimeError("no such file")
def get_trace_text(self, task_id):
return "[ACTION] fake probe\n[OBSERVATION]\n" + self.tasks[task_id].result_text
@@ -192,6 +213,22 @@ class TestParseHelpers(unittest.TestCase):
self.assertIsNone(parse_choice("no choice here"))
self.assertEqual(parse_why("<why>because</why>"), "because")
self.assertEqual(parse_notes("<notes>did x</notes>"), "did x")
def test_parse_deliverable_files_lenient(self):
text = ("=== DELIVERABLE FILE: a.md ===\nhello\n"
"=== END DELIVERABLE FILE ===\n"
"=== DELIVERABLE FILE: ../evil ===\nx\n"
"=== END DELIVERABLE FILE ===\n"
"=== DELIVERABLE FILE: b.md ===\nunterminated\n")
self.assertEqual(parse_deliverable_files(text), {"a.md": "hello\n"})
many = "".join(
f"=== DELIVERABLE FILE: f{i}.md ===\nx\n"
"=== END DELIVERABLE FILE ===\n" for i in range(10))
self.assertEqual(len(parse_deliverable_files(many)), 8)
big = ("=== DELIVERABLE FILE: big.md ===\n" + "x" * 70000 + "\n"
"=== END DELIVERABLE FILE ===\n")
content = parse_deliverable_files(big)["big.md"]
self.assertTrue(content.endswith("...[truncated]"))
exp = parse_experience(
"<experience>\ncue: c\nlesson: l\nevidence: e\nconfidence: high\n</experience>")
self.assertEqual((exp["cue"], exp["lesson"], exp["evidence"]), ("c", "l", "e"))
@@ -238,6 +275,109 @@ class TestFullRunViaTasks(unittest.TestCase):
self.assertIsNotNone(result.library_entry)
self.assertTrue((root / "library" / "accounting" / "INDEX.md").is_file())
def _models_by_role(self, client):
roles = {}
for key, tid in client.by_key.items():
for role in ("probe", "edit", "select", "reflect"):
if f"-{role}-" in key or key.endswith(f"-{role}"):
roles.setdefault(role, set()).add(client.task_models[tid])
return roles
def test_role_models_land_on_each_role(self):
client = FakeTaskClient()
tmp = tempfile.TemporaryDirectory()
self.addCleanup(tmp.cleanup)
root = Path(tmp.name)
result = run_improvement_via_tasks(
client, project_id="p1", task_id="ledger-1",
prompt="Sum every signed amount in ledger.csv. Return Total: with two decimals.",
base_harness=dict(BASE_TREE), run_tag="roles1",
work_dir=root / "work", domain="accounting",
source_files={"ledger.csv": LEDGER},
k=2, mode="parallel", library_root=root / "library",
model="shared-model",
role_models={"probe": "probe-model", "editor": "editor-model",
"selector": "selector-model",
"reflector": "reflector-model"})
self.assertTrue(result.ok)
roles = self._models_by_role(client)
self.assertEqual(roles["probe"], {"probe-model"})
self.assertEqual(roles["edit"], {"editor-model"})
self.assertEqual(roles["select"], {"selector-model"})
self.assertEqual(roles["reflect"], {"reflector-model"})
def test_role_model_falls_back_to_shared_then_default(self):
client = FakeTaskClient()
tmp = tempfile.TemporaryDirectory()
self.addCleanup(tmp.cleanup)
root = Path(tmp.name)
result = run_improvement_via_tasks(
client, project_id="p1", task_id="ledger-1",
prompt="Sum every signed amount in ledger.csv. Return Total: with two decimals.",
base_harness=dict(BASE_TREE), run_tag="roles2",
work_dir=root / "work", domain="accounting",
source_files={"ledger.csv": LEDGER},
k=1, mode="parallel", library_root=root / "library",
model="shared-model",
role_models={"editor": "editor-model"})
self.assertTrue(result.ok)
roles = self._models_by_role(client)
self.assertEqual(roles["probe"], {"shared-model"})
self.assertEqual(roles["edit"], {"editor-model"})
self.assertEqual(roles["reflect"], {"shared-model"})
# 两者都空时不传模型(服务端默认),假客户端记为 None。
client2 = FakeTaskClient()
result2, _, _ = self._run(client2, run_tag="roles3")
self.assertTrue(result2.ok)
self.assertEqual(set(client2.task_models.values()), {None})
def test_probe_deliverable_files_from_endpoint(self):
client = FakeTaskClient(endpoint_files={
"report.md": "# R\n",
"ledger.csv": "amount\n999.00\n",
"AGENTS.md": BASE_TREE["AGENTS.md"],
})
result, _, root = self._run(client, run_tag="assets1")
self.assertTrue(result.ok)
agg = result.mode_result.rollouts["agg1"]
# 新增文件与改动过的 seed 文件进产物;内容与 seed 一致的不进。
self.assertEqual(agg.deliverable_files,
{"report.md": "# R\n",
"ledger.csv": "amount\n999.00\n"})
found = [p for p in root.rglob("report.md")
if "deliverables/agg1/files" in str(p)]
self.assertTrue(found)
self.assertEqual(found[0].read_text(encoding="utf-8"), "# R\n")
def test_probe_deliverable_files_from_reply_blocks(self):
block = ("\n=== DELIVERABLE FILE: page.html ===\n<h1>hi</h1>\n"
"=== END DELIVERABLE FILE ===\n")
client = FakeTaskClient(endpoint_error=True, probe_extra=block)
result, _, _ = self._run(client, run_tag="assets2")
self.assertTrue(result.ok)
self.assertEqual(
result.mode_result.rollouts["agg1"].deliverable_files,
{"page.html": "<h1>hi</h1>\n"})
def test_probe_deliverable_files_endpoint_wins_on_collision(self):
block = ("\n=== DELIVERABLE FILE: page.html ===\nfrom-reply\n"
"=== END DELIVERABLE FILE ===\n")
client = FakeTaskClient(
endpoint_files={"page.html": "from-endpoint\n"},
probe_extra=block)
result, _, _ = self._run(client, run_tag="assets3")
self.assertTrue(result.ok)
self.assertEqual(
result.mode_result.rollouts["agg1"].deliverable_files,
{"page.html": "from-endpoint\n"})
def test_probe_deliverable_files_endpoint_error_is_not_fatal(self):
client = FakeTaskClient(endpoint_error=True)
result, _, _ = self._run(client, run_tag="assets4")
self.assertTrue(result.ok)
self.assertEqual(
result.mode_result.rollouts["agg1"].deliverable_files, {})
def test_same_run_tag_replays_without_new_tasks(self):
client = FakeTaskClient()
result1, _, _ = self._run(client, work_sub="work1")
@@ -563,5 +703,56 @@ class TestHttpClientRetry(unittest.TestCase):
self.assertEqual(counts["n"], 1)
class TestSweep(unittest.TestCase):
def _spec(self, tag, **over):
spec = {
"run_tag": tag, "task_id": "ledger-1",
"prompt": "Sum every signed amount in ledger.csv. Return Total: with two decimals.",
"base_harness": dict(BASE_TREE),
"source_files": {"ledger.csv": LEDGER},
"k": 1, "mode": "parallel", "domain": "accounting",
}
spec.update(over)
return spec
def test_sweep_isolates_failures_and_shares_library(self):
client = FakeTaskClient(fail_tags=("bad1",))
tmp = tempfile.TemporaryDirectory()
self.addCleanup(tmp.cleanup)
root = Path(tmp.name)
report = run_sweep(
client, project_id="p1",
runs=[self._spec("good1"), self._spec("bad1"),
self._spec("badk", k=99), self._spec("good2")],
work_root=root / "sweep", library_root=root / "library")
by_tag = {e["run_tag"]: e for e in report["runs"]}
self.assertTrue(by_tag["good1"]["ok"])
self.assertEqual(by_tag["good1"]["chosen_label"], "agg1")
# bad1:全部任务注入失败 → 轮次 ok=False,但不中断后续轮。
self.assertFalse(by_tag["bad1"]["ok"])
# badk:k 越界在方法层抛错 → 记入 error,同样不中断。
self.assertFalse(by_tag["badk"]["ok"])
self.assertIn("ValueError", by_tag["badk"]["error"])
self.assertTrue(by_tag["good2"]["ok"])
self.assertEqual(report["totals"]["n_runs"], 4)
self.assertEqual(report["totals"]["n_ok"], 2)
# 共享库:两轮好结果都入库,同一 INDEX 可见。
self.assertIsNotNone(by_tag["good1"]["library_entry"])
self.assertIsNotNone(by_tag["good2"]["library_entry"])
self.assertTrue((root / "library" / "accounting" / "INDEX.md").is_file())
def test_sweep_requires_run_tag(self):
client = FakeTaskClient()
tmp = tempfile.TemporaryDirectory()
self.addCleanup(tmp.cleanup)
root = Path(tmp.name)
spec = self._spec("good1")
del spec["run_tag"]
report = run_sweep(client, project_id="p1", runs=[spec],
work_root=root / "sweep")
self.assertFalse(report["runs"][0]["ok"])
self.assertIn("run_tag", report["runs"][0]["error"])
if __name__ == "__main__":
unittest.main()
+165
View File
@@ -0,0 +1,165 @@
"""GET /tasks/{id}/files 端点测试:本地 worktree 文件读取。
只用标准库 http.client,不跑真实模型,不联网。
任务行直接插库(不走 POST /tasks,避免 worker 派发),
worktree 用 server.host_worktrees_dir 注入到临时目录。
"""
import http.client
import json
import os
import sqlite3
import sys
import tempfile
import threading
import unittest
from pathlib import Path
from urllib.parse import quote
try:
from cloud.auth import AuthStore
from cloud.server import create_server
except ImportError:
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
from cloud.auth import AuthStore
from cloud.server import create_server
class TaskFilesTestCase(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.db_path = os.path.join(self.tmp.name, "control.db")
self.auth = AuthStore(self.db_path)
self.owner_id, self.owner_key = self.auth.create_user("owner")
self.viewer_id, self.viewer_key = self.auth.create_user("viewer")
self.outsider_id, self.outsider_key = self.auth.create_user("outsider")
self.server = create_server(self.db_path, "127.0.0.1", 0)
self.worktrees = os.path.join(self.tmp.name, "host-worktrees")
os.makedirs(self.worktrees)
self.server.host_worktrees_dir = self.worktrees
self.port = self.server.server_address[1]
self.thread = threading.Thread(
target=self.server.serve_forever, daemon=True)
self.thread.start()
status, self.project = self._req(
"POST", "/projects", {"name": "demo"}, self.owner_key)
self.assertEqual(status, 201)
self.auth.set_member(
self.project["id"], self.owner_id, self.viewer_id, "viewer")
self._task("task-files-1", executor=None)
wt = os.path.join(self.worktrees, "task-files-1")
os.makedirs(os.path.join(wt, "sub"))
Path(wt, "report.md").write_text("# Report\n", encoding="utf-8")
Path(wt, "sub", "page.html").write_text("<h1>hi</h1>\n",
encoding="utf-8")
# 非法 UTF-8 字节(\x00 是合法 UTF-8,不能用来测二进制拒绝)。
Path(wt, "blob.bin").write_bytes(b"\x89PNG\r\n\x1a\n\xff\xfe")
Path(wt, "big.txt").write_text("x" * (256 * 1024 + 1),
encoding="utf-8")
os.symlink(os.path.join(self.tmp.name, "outside.txt"),
os.path.join(wt, "escape"))
Path(self.tmp.name, "outside.txt").write_text("secret\n",
encoding="utf-8")
self._task("task-docker-1", executor="docker")
self._task("task-nowt-1", executor=None)
def tearDown(self):
self.server.shutdown()
self.server.server_close()
self.thread.join(timeout=5)
try:
self.auth.conn.close()
except Exception:
pass
self.tmp.cleanup()
def _task(self, task_id, executor):
conn = sqlite3.connect(self.db_path)
try:
conn.execute(
"INSERT INTO tasks(id, project_id, prompt, status, executor)"
" VALUES(?, ?, ?, ?, ?)",
(task_id, self.project["id"], "do it", "completed", executor))
conn.commit()
finally:
conn.close()
def _req(self, method, path, body=None, key=None):
conn = http.client.HTTPConnection("127.0.0.1", self.port, timeout=10)
hdrs = {}
payload = None
if body is not None:
payload = json.dumps(body).encode("utf-8")
hdrs["Content-Type"] = "application/json"
if key is not None:
hdrs["X-API-Key"] = key
try:
conn.request(method, path, body=payload, headers=hdrs)
resp = conn.getresponse()
data = resp.read()
finally:
conn.close()
try:
parsed = json.loads(data.decode("utf-8")) if data else None
except ValueError:
parsed = None
return resp.status, parsed
def _files(self, task_id, path=None, key=None):
url = f"/tasks/{task_id}/files"
if path is not None:
url += "?path=" + quote(path)
return self._req("GET", url, key=self.owner_key if key is None else key)
def test_list_top_level_recursive(self):
status, body = self._files("task-files-1")
self.assertEqual(status, 200, body)
paths = sorted(e["path"] for e in body["entries"])
self.assertIn("report.md", paths)
self.assertIn("sub/page.html", paths)
self.assertNotIn("escape", paths) # 符号链接不跟进列表
def test_read_file_and_subdir_listing(self):
status, body = self._files("task-files-1", "report.md")
self.assertEqual(status, 200, body)
self.assertEqual(body["content"], "# Report\n")
status, body = self._files("task-files-1", "sub")
self.assertEqual(status, 200, body)
self.assertEqual([e["path"] for e in body["entries"]],
["sub/page.html"])
def test_path_escape_blocked(self):
for bad in ("../outside.txt", "/etc/passwd", "escape", "sub/../../x"):
status, body = self._files("task-files-1", bad)
self.assertIn(status, (400, 404), (bad, body))
# escape 是指到 worktree 外的符号链接:必须 400,不是读到 secret。
status, body = self._files("task-files-1", "escape")
self.assertEqual(status, 400, body)
def test_missing_file_and_missing_worktree(self):
status, _ = self._files("task-files-1", "nope.md")
self.assertEqual(status, 404)
status, _ = self._files("task-nowt-1")
self.assertEqual(status, 404)
status, _ = self._files("task-unknown")
self.assertEqual(status, 404)
def test_binary_and_oversized(self):
status, body = self._files("task-files-1", "blob.bin")
self.assertEqual(status, 409, body)
status, body = self._files("task-files-1", "big.txt")
self.assertEqual(status, 413, body)
def test_docker_task_409_and_auth(self):
status, body = self._files("task-docker-1")
self.assertEqual(status, 409, body)
status, _ = self._req("GET", "/tasks/task-files-1/files")
self.assertEqual(status, 401)
status, _ = self._files("task-files-1", key=self.outsider_key)
self.assertEqual(status, 403)
status, _ = self._files("task-files-1", key=self.viewer_key)
self.assertEqual(status, 200)
if __name__ == "__main__":
unittest.main()