harness2: per-role models, sweep batch, task file assets endpoint
- Role models: run_improvement_via_tasks gains role_models
(probe/editor/selector/reflector) plus --model-<role> CLI flags;
missing/empty falls back to shared --model, both empty means
server default, so default behavior is unchanged.
- Sweep: run_sweep + examples/harness2_sweep.py manifest CLI with
shared library_root (library restore), per-run failure isolation
and sweep-report.json summary; preflight gates task creation.
- Deliverable assets: GET /tasks/{id}/files for local-executor
tasks (safe path resolution, UTF-8 text <=256KB, listing <=200;
docker/cloud/cross-host 409 with explicit reason, since tenant
Docker shares one /app/work per project and per-task attribution
would misattribute); probe rollout deliverable_files from the
endpoint plus DELIVERABLE FILE reply blocks (endpoint wins on
collision, bad blocks dropped), surfaced in
deliverables/<label>/files/ context and full_result summary.
- Docs: harness2.md comparison table updated for all three items.
This commit is contained in:
1 parent
9e444eca38
commit
c20192bce8
8 files changed
+856
-28
No files matched your search
+58
-1
@@ -241,6 +241,56 @@ def parse_harness_files(text: str) -> dict[str, str]:
|
||||
return out
|
||||
|
||||
|
||||
_DELIVERABLE_FILE_START_RX = re.compile(r"^=== DELIVERABLE FILE:\s*(.+?)\s*===$")
|
||||
_DELIVERABLE_FILE_END = "=== END DELIVERABLE FILE ==="
|
||||
_DELIVERABLE_FILE_MAX = 8
|
||||
_DELIVERABLE_FILE_MAX_CHARS = 65536
|
||||
|
||||
|
||||
def _is_safe_deliverable_path(path: str) -> bool:
|
||||
if not path or path.startswith("/") or "\x00" in path:
|
||||
return False
|
||||
return all(p not in ("", ".", "..") for p in path.split("/"))
|
||||
|
||||
|
||||
def parse_deliverable_files(text: str) -> dict[str, str]:
|
||||
"""解析探针回复里的产物文件块,返回 {相对路径: 内容}。
|
||||
|
||||
与编辑文件块的严格语义不同:这里故意宽容——坏块(未闭合、
|
||||
非法路径、超量、超长)只丢弃该块、不抛错,因为产物文件是
|
||||
交付文本之外的 enrichment,绝不能因为一块坏内容毁掉整轮
|
||||
交付物。超长内容截断并打截断标记,让评审知道它不完整;
|
||||
超过 8 块时只取前 8 块。同一路径出现两次时后一块覆盖前一块。
|
||||
"""
|
||||
out: dict[str, str] = {}
|
||||
cur_path: str | None = None
|
||||
cur_lines: list[str] = []
|
||||
for line in (text or "").splitlines():
|
||||
m = _DELIVERABLE_FILE_START_RX.match(line.strip())
|
||||
if m:
|
||||
if cur_path is not None:
|
||||
cur_path = None
|
||||
cur_lines = []
|
||||
path = m.group(1).strip()
|
||||
if _is_safe_deliverable_path(path) and len(out) < _DELIVERABLE_FILE_MAX:
|
||||
cur_path = path
|
||||
cur_lines = []
|
||||
continue
|
||||
if line.strip() == _DELIVERABLE_FILE_END:
|
||||
if cur_path is not None:
|
||||
content = "\n".join(cur_lines) + ("\n" if cur_lines else "")
|
||||
if len(content) > _DELIVERABLE_FILE_MAX_CHARS:
|
||||
content = (content[:_DELIVERABLE_FILE_MAX_CHARS]
|
||||
+ "\n...[truncated]")
|
||||
out[cur_path] = content
|
||||
cur_path = None
|
||||
cur_lines = []
|
||||
continue
|
||||
if cur_path is not None:
|
||||
cur_lines.append(line)
|
||||
return out
|
||||
|
||||
|
||||
MAX_K = 3
|
||||
STAGE_ATTEMPTS = 2 # 提议被拒后给一次按反馈修正的机会,再拒就回落
|
||||
|
||||
@@ -490,6 +540,7 @@ class Rollout:
|
||||
trajectory: str = ""
|
||||
cost_usd: float = 0.0
|
||||
error: str = ""
|
||||
deliverable_files: dict[str, str] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -589,7 +640,12 @@ def build_context(
|
||||
(dd / f"{r.label}.md").write_text(
|
||||
r.deliverable_text or "(no deliverable)", encoding="utf-8"
|
||||
)
|
||||
lines.append("context/deliverables/ what each rollout produced")
|
||||
for name, content in (r.deliverable_files or {}).items():
|
||||
dest = dd / r.label / "files" / name
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
dest.write_text(content, encoding="utf-8")
|
||||
lines.append("context/deliverables/ what each rollout produced"
|
||||
" (+ <label>/files/ when it produced files)")
|
||||
write_tree(harness, root / "harness")
|
||||
lines.append("context/harness/ the harness tree you may edit, and ONLY this")
|
||||
if library_dir and Path(library_dir).is_dir():
|
||||
@@ -842,6 +898,7 @@ class ModeResult:
|
||||
"delivered": r.delivered,
|
||||
"error": (r.error or "")[:400],
|
||||
"deliverable_text": (r.deliverable_text or "")[:500],
|
||||
"deliverable_files": sorted(r.deliverable_files or {}),
|
||||
}
|
||||
for lbl, r in self.rollouts.items()
|
||||
},
|
||||
|
||||
+145
-13
@@ -1,6 +1,6 @@
|
||||
"""Harness² 接线层:把 cloud/harness2.py 的三个协议落到真实 codex 任务上。
|
||||
|
||||
只用控制面已有的 API,不新增任何服务端端点:
|
||||
控制面 API:
|
||||
POST /uploads(multipart + paths JSON)-> upload_id
|
||||
POST /tasks(upload_id 绑定 + idempotency_key 幂等)-> 任务排队
|
||||
GET /tasks/{id} 轮询终态(completed / failed / interrupted /
|
||||
@@ -8,6 +8,8 @@
|
||||
GET /tasks/{id}/trace 轨迹(探针的 trajectory 来源)
|
||||
GET /tasks/{id}/cost 成本(usd_micros / 1e6;无价 token 不计入 USD,
|
||||
这是现有 cost 模型的边界,不是本层漏算)
|
||||
GET /tasks/{id}/files 读本地任务 worktree 文件(产物资产端点;
|
||||
Docker/跨机任务回 409,见下)
|
||||
POST /tasks/{id}/cancel 超时取消,不留孤儿任务继续烧钱
|
||||
认证:X-API-Key 请求头;密钥只由调用方持有,本模块不读、不存、
|
||||
不打印密钥。
|
||||
@@ -18,12 +20,13 @@
|
||||
文件会覆盖仓库同名文件——这正是探针要的效果:同一任务在不同
|
||||
harness 下跑。prompt = 原任务 prompt + 一段交付脚注(见
|
||||
_PROBE_DELIVERABLE_FOOTER),因为评审只看最终回复:agent 必须
|
||||
把交付物完整写在最终回复里,不能只说“文件已写好”。这就是本层
|
||||
最大的适用边界:交付物是最终回复文本的任务;文件型产物(代码
|
||||
diff、报告文件)目前只能靠最终回复复述/摘要,worktree 快照 API
|
||||
只给 diff 截断与未跟踪文件列表、不给完整文件内容,跨主机还读
|
||||
不到——文件产物评审需要新增文件读取端点,另行评审,不在本层
|
||||
假装已解决。
|
||||
把交付物完整写在最终回复里,不能只说“文件已写好”。
|
||||
文件型产物走两层(产物资产):①本地任务经 GET /tasks/{id}/files
|
||||
读 worktree 新增/改动文件(端点为准);②所有执行器都可用
|
||||
回复里的 DELIVERABLE FILE 块(agent 自报,端点结果优先)。
|
||||
边界仍在:租户 Docker 下 /app/work 是整项目共享目录,按任务
|
||||
归属读文件会错指,端点对 Docker 任务一律 409——这时只有第
|
||||
②层可用,且内容以 agent 自报为准(可能截断,截断有标记)。
|
||||
- 编辑(CodexEditor.edit):bundle = context/ 前缀的证据树(task.md /
|
||||
rollouts / deliverables / harness / library / pool / decisions)。
|
||||
agent 在 worktree 的 context/harness/ 里改文件,最终回复里用
|
||||
@@ -65,6 +68,7 @@ import json
|
||||
import re
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
import uuid
|
||||
from dataclasses import dataclass, field
|
||||
@@ -77,6 +81,7 @@ from .harness2 import (
|
||||
Tree,
|
||||
improve_task,
|
||||
parse_choice,
|
||||
parse_deliverable_files,
|
||||
parse_experience,
|
||||
parse_harness_files,
|
||||
parse_no_change,
|
||||
@@ -185,6 +190,11 @@ write the complete deliverable in your final reply itself — do not only say
|
||||
"done" or "the file is written". If the task asks for a file, also write the
|
||||
file in the worktree, but the final reply must still contain the full content
|
||||
a reviewer needs to judge the work.
|
||||
If you created deliverable files (a page, a report, an image manifest),
|
||||
also paste each file's full content in your final reply as a block:
|
||||
=== DELIVERABLE FILE: <relative path> ===
|
||||
<complete file content>
|
||||
=== END DELIVERABLE FILE ===
|
||||
"""
|
||||
|
||||
_EDITOR_OUTPUT_FORMAT = """
|
||||
@@ -278,6 +288,12 @@ class TaskClient(Protocol):
|
||||
"""轨迹渲染文本;读不到时返回空串(调用方用最终回复兜底)。"""
|
||||
...
|
||||
|
||||
def get_task_files(self, task_id: str, path: str = "") -> dict:
|
||||
"""GET /tasks/{id}/files:不传 path 列 worktree 文件,传 path
|
||||
读单个文件。服务端对 Docker/跨机任务回 409,调用方须把
|
||||
409 当成“无文件通道”、不得当成探针失败。"""
|
||||
...
|
||||
|
||||
def get_cost_usd(self, task_id: str) -> float:
|
||||
"""有价成本 USD;读不到时返回 0.0,不炸轮次。"""
|
||||
...
|
||||
@@ -452,6 +468,10 @@ class HttpTaskClient:
|
||||
rendered = "\n".join(lines).strip()
|
||||
return rendered[:TRACE_MAX_CHARS] if rendered else ""
|
||||
|
||||
def get_task_files(self, task_id: str, path: str = "") -> dict:
|
||||
q = f"?path={urllib.parse.quote(path)}" if path else ""
|
||||
return self._request("GET", f"/tasks/{task_id}/files{q}")
|
||||
|
||||
def get_cost_usd(self, task_id: str) -> float:
|
||||
try:
|
||||
obj = self._request("GET", f"/tasks/{task_id}/cost")
|
||||
@@ -596,6 +616,12 @@ class CodexProbeAdapter:
|
||||
error=f"{type(exc).__name__}: {exc}"[:400])
|
||||
state = done.state
|
||||
delivered = state.status == "completed" and bool(state.result_text.strip())
|
||||
deliverable_files = self._collect_deliverable_files(done.task_id, files)
|
||||
# 回复里的产物文件块是补充通道(所有执行器都可用);
|
||||
# 端点读到的文件为准,冲突时不覆盖端点结果。
|
||||
for name, content in parse_deliverable_files(
|
||||
state.result_text or "").items():
|
||||
deliverable_files.setdefault(name, content)
|
||||
return Rollout(
|
||||
label=label, ok=state.status == "completed", delivered=delivered,
|
||||
deliverable_text=state.result_text,
|
||||
@@ -603,8 +629,46 @@ class CodexProbeAdapter:
|
||||
cost_usd=done.cost_usd,
|
||||
error="" if state.status == "completed" else (
|
||||
state.error_text or f"task ended as {state.status}")[:400],
|
||||
deliverable_files=deliverable_files,
|
||||
)
|
||||
|
||||
def _collect_deliverable_files(self, task_id: str,
|
||||
seed_files: dict[str, bytes]) -> dict[str, str]:
|
||||
"""经 GET /tasks/{id}/files 读探针 worktree 里新增的文件。
|
||||
|
||||
取 seed 里没有的新文件,以及内容被改过的 seed 文件(与
|
||||
seed 字节比对,不一致才算产物);跳过隐藏文件、最多收
|
||||
8 个、最多读 16 个;端点不可用(Docker/跨机 409、读错)
|
||||
时返回空 dict——文件产物是 enrichment,绝不能因此炸轮次。
|
||||
"""
|
||||
try:
|
||||
listing = self.client.get_task_files(task_id)
|
||||
except Exception:
|
||||
return {}
|
||||
out: dict[str, str] = {}
|
||||
reads = 0
|
||||
for entry in listing.get("entries") or []:
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
path = entry.get("path") or ""
|
||||
if not path or path.startswith("."):
|
||||
continue
|
||||
if len(out) >= 8 or reads >= 16:
|
||||
break
|
||||
try:
|
||||
data = self.client.get_task_files(task_id, path)
|
||||
except Exception:
|
||||
continue
|
||||
reads += 1
|
||||
content = data.get("content")
|
||||
if not isinstance(content, str):
|
||||
continue
|
||||
seed = seed_files.get(path)
|
||||
if seed is not None and seed.decode("utf-8", errors="replace") == content:
|
||||
continue
|
||||
out[path] = content
|
||||
return out
|
||||
|
||||
|
||||
@dataclass
|
||||
class CodexEditor:
|
||||
@@ -773,6 +837,7 @@ def run_improvement_via_tasks(
|
||||
budget_usd: float | None = None,
|
||||
max_sessions: int | None = None,
|
||||
model: str | None = None,
|
||||
role_models: dict[str, str | None] | None = None,
|
||||
effort: str | None = None,
|
||||
approval: str | None = None,
|
||||
environment_id: str | None = None,
|
||||
@@ -785,22 +850,37 @@ def run_improvement_via_tasks(
|
||||
library_root 为空时不建经验库(只跑改进、不入库);非空时按
|
||||
domain 建库。auto_apply 恒为 False——采纳的树只入库待评审,
|
||||
不自动成为后续任务的 base(见 docs/harness2.md 安全一节)。
|
||||
|
||||
分角色模型:role_models 键为 probe / editor / selector /
|
||||
reflector,非空值覆盖该角色的 model;缺键或空值时回落到
|
||||
共享 model;两者都空时不传 model(服务端默认模型)。默认
|
||||
行为与此前完全一致——本参数只提供分流能力,不替用户选
|
||||
模型(有价模型会产生真实成本,见 docs/harness2.md)。
|
||||
"""
|
||||
from .harness2 import Library
|
||||
|
||||
roles = role_models or {}
|
||||
|
||||
def _role_model(role: str) -> str | None:
|
||||
return roles.get(role) or model
|
||||
|
||||
adapter = CodexProbeAdapter(
|
||||
client=client, project_id=project_id, task_id=task_id, prompt=prompt,
|
||||
base=base_harness, run_tag=run_tag,
|
||||
source_files=source_files or {}, environment_id=environment_id,
|
||||
executor=executor, model=model, effort=effort, approval=approval,
|
||||
executor=executor, model=_role_model("probe"), effort=effort,
|
||||
approval=approval,
|
||||
no_time_limit=no_time_limit, timeout_seconds=timeout_seconds,
|
||||
domain=domain)
|
||||
shared = dict(environment_id=environment_id, executor=executor, model=model,
|
||||
shared = dict(environment_id=environment_id, executor=executor,
|
||||
effort=effort, approval=approval, no_time_limit=no_time_limit,
|
||||
timeout_seconds=timeout_seconds)
|
||||
editor = CodexEditor(client=client, project_id=project_id, run_tag=run_tag, **shared)
|
||||
selector = CodexSelector(client=client, project_id=project_id, run_tag=run_tag, **shared)
|
||||
reflector = CodexReflector(client=client, project_id=project_id, run_tag=run_tag, **shared)
|
||||
editor = CodexEditor(client=client, project_id=project_id, run_tag=run_tag,
|
||||
model=_role_model("editor"), **shared)
|
||||
selector = CodexSelector(client=client, project_id=project_id, run_tag=run_tag,
|
||||
model=_role_model("selector"), **shared)
|
||||
reflector = CodexReflector(client=client, project_id=project_id, run_tag=run_tag,
|
||||
model=_role_model("reflector"), **shared)
|
||||
library = Library(library_root, domain) if library_root is not None else None
|
||||
validate_run_tag(run_tag)
|
||||
return improve_task(
|
||||
@@ -809,10 +889,62 @@ def run_improvement_via_tasks(
|
||||
auto_apply=False, budget_usd=budget_usd, max_sessions=max_sessions)
|
||||
|
||||
|
||||
def run_sweep(client, *, project_id, runs, work_root, library_root=None):
|
||||
"""批量跑多轮改进:共享经验库、单轮失败隔离、汇总报告。
|
||||
|
||||
runs:每轮一个 dict,键为 run_improvement_via_tasks 的参数,
|
||||
run_tag 必填;client / project_id / work_dir / library_root
|
||||
由本函数统管,出现在 spec 里会被忽略(不报错)。每轮
|
||||
work_dir = work_root/<run_tag>。library_root 非空时所有轮
|
||||
共享同一库——已入库的经验会被后续轮读到,即上游 sweep 的
|
||||
library restore 语义;为空时各轮都不入库(仍汇总)。
|
||||
|
||||
单轮抛错只记入报告的 error,不中断后续轮(dead 隔离)。
|
||||
返回报告 dict:runs 逐轮(ok / chosen_label / deliverable
|
||||
前 500 字 / cost / library_entry / error)+ totals(轮数、
|
||||
成功轮数、总成本)。
|
||||
"""
|
||||
work_root = Path(work_root)
|
||||
report_runs = []
|
||||
total_cost = 0.0
|
||||
n_ok = 0
|
||||
for spec in runs:
|
||||
spec = dict(spec)
|
||||
run_tag = spec.pop("run_tag", "")
|
||||
for owned in ("client", "project_id", "work_dir", "library_root"):
|
||||
spec.pop(owned, None)
|
||||
entry = {"run_tag": run_tag, "ok": False, "chosen_label": None,
|
||||
"deliverable_text": "", "total_cost_usd": 0.0,
|
||||
"library_entry": None, "error": None}
|
||||
try:
|
||||
if not run_tag:
|
||||
raise ValueError("sweep 每轮必须给 run_tag(用作 work_dir 子目录)")
|
||||
validate_run_tag(run_tag)
|
||||
result = run_improvement_via_tasks(
|
||||
client, project_id=project_id, run_tag=run_tag,
|
||||
work_dir=work_root / run_tag,
|
||||
library_root=library_root, **spec)
|
||||
entry["ok"] = bool(result.ok)
|
||||
entry["chosen_label"] = result.selection.chosen_label
|
||||
chosen = result.mode_result.rollouts.get(result.selection.chosen_label)
|
||||
if chosen is not None:
|
||||
entry["deliverable_text"] = (chosen.deliverable_text or "")[:500]
|
||||
entry["total_cost_usd"] = result.mode_result.total_cost_usd
|
||||
entry["library_entry"] = result.library_entry
|
||||
total_cost += result.mode_result.total_cost_usd
|
||||
n_ok += 1 if result.ok else 0
|
||||
except Exception as exc:
|
||||
entry["error"] = f"{type(exc).__name__}: {exc}"[:400]
|
||||
report_runs.append(entry)
|
||||
return {"runs": report_runs,
|
||||
"totals": {"n_runs": len(report_runs), "n_ok": n_ok,
|
||||
"total_cost_usd": total_cost}}
|
||||
|
||||
|
||||
__all__ = [
|
||||
"TERMINAL_STATUSES", "DEFAULT_TASK_TIMEOUT_SECONDS", "DEFAULT_POLL_SECONDS",
|
||||
"TRACE_MAX_CHARS", "TaskState", "TaskClient", "build_multipart",
|
||||
"HttpTaskClient", "CompletedTask", "run_task",
|
||||
"CodexProbeAdapter", "CodexEditor", "CodexSelector", "CodexReflector",
|
||||
"run_improvement_via_tasks", "validate_run_tag", "preflight_checks",
|
||||
"run_improvement_via_tasks", "run_sweep", "validate_run_tag", "preflight_checks",
|
||||
]
|
||||
@@ -445,6 +445,22 @@ def _thread_worktree_for_task(server, task):
|
||||
return os.path.join(base, safe)
|
||||
|
||||
|
||||
def _safe_worktree_path(worktree, rel):
|
||||
"""worktree 内相对路径的安全解析:空串返回 worktree 本身;
|
||||
绝对路径、.. 逃逸、符号链接逃逸一律返回 None。realpath 后
|
||||
再校验 commonpath,目录内的符号链接指回内部时放行(读到
|
||||
的仍是 worktree 内的内容),指出去时拦住。"""
|
||||
if not rel:
|
||||
return worktree
|
||||
if os.path.isabs(rel):
|
||||
return None
|
||||
base = os.path.realpath(worktree)
|
||||
target = os.path.realpath(os.path.join(base, rel))
|
||||
if os.path.commonpath([base, target]) != base:
|
||||
return None
|
||||
return target
|
||||
|
||||
|
||||
def _extract_thread_list(result):
|
||||
"""从 thread-list 的 CLI 结果里取 thread 列表:标准形态是
|
||||
ThreadListResponse 的 data;threads 键与 result 包裹是
|
||||
@@ -2152,6 +2168,8 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
|
||||
self._list_followups(user_id, rest[: -len("/followups")])
|
||||
elif rest.endswith("/timeline") and method == "GET":
|
||||
self._task_timeline(user_id, rest[: -len("/timeline")])
|
||||
elif rest.endswith("/files") and method == "GET":
|
||||
self._task_files(user_id, rest[: -len("/files")], query)
|
||||
elif rest.endswith("/chat-history") and method == "GET":
|
||||
self._task_chat_history(
|
||||
user_id, rest[: -len("/chat-history")])
|
||||
@@ -4698,6 +4716,87 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
|
||||
for r in cp_rows],
|
||||
})
|
||||
|
||||
def _task_files(self, user_id, task_id, query):
|
||||
"""GET /tasks/{id}/files:读本地任务 worktree 里的文件。
|
||||
|
||||
只做 local 执行器任务(复用 thread 操作的执行器门):
|
||||
Docker 任务的 worktree 在容器内、且租户 Docker 下 /app/work
|
||||
是整项目共享目录,按任务归属读文件会错指,一律 409 明
|
||||
说;cloud 任务与跨机 worktree 同理。executor 为空的老
|
||||
任务放行(与 thread 门同款),解析不到 worktree 时 404。
|
||||
|
||||
?path=<相对路径>:不传列 worktree 顶层(递归、最多 200
|
||||
条,只含文件路径与大小,不跟符号链接);传目录列该目录
|
||||
(非递归,含 is_dir);传文件读文本内容(UTF-8、≤256KB,
|
||||
二进制或超大一律 409/413 明说,不截断冒充全文)。路径
|
||||
解析一律走 _safe_worktree_path,防 .. 与符号链接逃逸。
|
||||
读操作不限制任务状态(进行中任务读到的是中间态)。
|
||||
"""
|
||||
task, err = self._thread_task(user_id, task_id, "read")
|
||||
if err is not None:
|
||||
self._send_json(err[0], {"error": err[1]})
|
||||
return
|
||||
guard = self._thread_task_guard(task)
|
||||
if guard is not None:
|
||||
self._send_json(guard[0], {"error": guard[1]})
|
||||
return
|
||||
worktree = _thread_worktree_for_task(self.server, task)
|
||||
if not os.path.isdir(worktree):
|
||||
self._send_json(404, {"error": "no local worktree for this task"})
|
||||
return
|
||||
rel = (query.get("path") or [""])[0]
|
||||
target = _safe_worktree_path(worktree, rel)
|
||||
if target is None:
|
||||
self._send_json(400, {"error": "path escapes task worktree"})
|
||||
return
|
||||
if os.path.isdir(target):
|
||||
entries = []
|
||||
if not rel:
|
||||
for root, dirs, files in os.walk(target):
|
||||
dirs[:] = [d for d in dirs
|
||||
if not os.path.islink(os.path.join(root, d))]
|
||||
dirs.sort()
|
||||
for name in sorted(files):
|
||||
fpath = os.path.join(root, name)
|
||||
if os.path.islink(fpath):
|
||||
continue
|
||||
entries.append({
|
||||
"path": os.path.relpath(fpath, worktree),
|
||||
"size": os.path.getsize(fpath),
|
||||
})
|
||||
if len(entries) >= 200:
|
||||
break
|
||||
if len(entries) >= 200:
|
||||
break
|
||||
else:
|
||||
for name in sorted(os.listdir(target)):
|
||||
fpath = os.path.join(target, name)
|
||||
is_link = os.path.islink(fpath)
|
||||
entries.append({
|
||||
"path": os.path.relpath(fpath, worktree),
|
||||
"size": os.path.getsize(fpath)
|
||||
if os.path.isfile(fpath) and not is_link else 0,
|
||||
"is_dir": os.path.isdir(fpath) and not is_link,
|
||||
})
|
||||
self._send_json(200, {"task_id": task_id, "path": rel,
|
||||
"entries": entries})
|
||||
return
|
||||
if not os.path.isfile(target):
|
||||
self._send_json(404, {"error": "no such file"})
|
||||
return
|
||||
size = os.path.getsize(target)
|
||||
if size > 256 * 1024:
|
||||
self._send_json(413, {"error": "file too large for inline read"})
|
||||
return
|
||||
try:
|
||||
with open(target, encoding="utf-8") as fh:
|
||||
content = fh.read()
|
||||
except (OSError, UnicodeDecodeError):
|
||||
self._send_json(409, {"error": "file is not readable UTF-8 text"})
|
||||
return
|
||||
self._send_json(200, {"task_id": task_id, "path": rel,
|
||||
"size": size, "content": content})
|
||||
|
||||
def _task_chat_history(self, user_id, task_id):
|
||||
"""GET /tasks/{id}/chat-history:真正的交替历史,一次性
|
||||
按时间交错拼好返回,前端不再猜顺序。
|
||||
|
||||
@@ -19,26 +19,26 @@ from pathlib import Path
|
||||
try:
|
||||
from cloud import harness2_tasks
|
||||
from cloud.harness2 import (
|
||||
parse_choice, parse_experience, parse_harness_files, parse_no_change,
|
||||
parse_notes, parse_why,
|
||||
parse_choice, parse_deliverable_files, parse_experience,
|
||||
parse_harness_files, parse_no_change, parse_notes, parse_why,
|
||||
)
|
||||
from cloud.harness2_tasks import (
|
||||
CodexEditor, CodexProbeAdapter, HttpTaskClient, TaskState,
|
||||
build_multipart, preflight_checks, run_improvement_via_tasks, run_task,
|
||||
unwrap_result_text, validate_run_tag,
|
||||
build_multipart, preflight_checks, run_improvement_via_tasks, run_sweep,
|
||||
run_task, unwrap_result_text, validate_run_tag,
|
||||
)
|
||||
from cloud.uploads import parse_multipart, parse_paths_field
|
||||
except ImportError: # discover -s cloud/tests 时把项目根目录补进 sys.path
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||||
from cloud import harness2_tasks
|
||||
from cloud.harness2 import (
|
||||
parse_choice, parse_experience, parse_harness_files, parse_no_change,
|
||||
parse_notes, parse_why,
|
||||
parse_choice, parse_deliverable_files, parse_experience,
|
||||
parse_harness_files, parse_no_change, parse_notes, parse_why,
|
||||
)
|
||||
from cloud.harness2_tasks import (
|
||||
CodexEditor, CodexProbeAdapter, HttpTaskClient, TaskState,
|
||||
build_multipart, preflight_checks, run_improvement_via_tasks, run_task,
|
||||
unwrap_result_text, validate_run_tag,
|
||||
build_multipart, preflight_checks, run_improvement_via_tasks, run_sweep,
|
||||
run_task, unwrap_result_text, validate_run_tag,
|
||||
)
|
||||
from cloud.uploads import parse_multipart, parse_paths_field
|
||||
|
||||
@@ -61,15 +61,22 @@ LEDGER = b"amount\n100.25\n-20.10\n5.00\n"
|
||||
class FakeTaskClient:
|
||||
"""进程内控制面:按 prompt 类型分派到确定性假 worker。"""
|
||||
|
||||
def __init__(self, editor_mode="fix", selector_mode="signed", envelope=False):
|
||||
def __init__(self, editor_mode="fix", selector_mode="signed", envelope=False,
|
||||
fail_tags=(), endpoint_files=None, endpoint_error=False,
|
||||
probe_extra=""):
|
||||
self.editor_mode = editor_mode
|
||||
self.selector_mode = selector_mode
|
||||
self.envelope = envelope
|
||||
self.fail_tags = tuple(fail_tags)
|
||||
self.endpoint_files = dict(endpoint_files or {})
|
||||
self.endpoint_error = endpoint_error
|
||||
self.probe_extra = probe_extra
|
||||
self.uploads = {}
|
||||
self.tasks = {}
|
||||
self.by_key = {}
|
||||
self.created = 0
|
||||
self.cancelled = []
|
||||
self.task_models = {}
|
||||
|
||||
def upload(self, files):
|
||||
uid = f"u{len(self.uploads) + 1}"
|
||||
@@ -78,6 +85,8 @@ class FakeTaskClient:
|
||||
|
||||
def create_task(self, *, project_id, prompt, upload_id=None,
|
||||
idempotency_key=None, **kwargs):
|
||||
if idempotency_key and any(t in idempotency_key for t in self.fail_tags):
|
||||
raise RuntimeError("injected task failure")
|
||||
if idempotency_key and idempotency_key in self.by_key:
|
||||
return self.by_key[idempotency_key]
|
||||
tid = f"t{len(self.tasks) + 1}"
|
||||
@@ -94,6 +103,7 @@ class FakeTaskClient:
|
||||
})
|
||||
self.tasks[tid] = TaskState(task_id=tid, status="completed",
|
||||
result_text=result)
|
||||
self.task_models[tid] = kwargs.get("model")
|
||||
if idempotency_key:
|
||||
self.by_key[idempotency_key] = tid
|
||||
return tid
|
||||
@@ -163,11 +173,22 @@ class FakeTaskClient:
|
||||
[sys.executable, str(root / "scripts" / "summarize.py"),
|
||||
str(root / "ledger.csv")],
|
||||
capture_output=True, text=True, timeout=10)
|
||||
return proc.stdout.strip()
|
||||
return proc.stdout.strip() + self.probe_extra
|
||||
|
||||
def get_task(self, task_id):
|
||||
return self.tasks[task_id]
|
||||
|
||||
def get_task_files(self, task_id, path=""):
|
||||
if self.endpoint_error:
|
||||
raise RuntimeError("409 files unavailable")
|
||||
if not path:
|
||||
return {"entries": [
|
||||
{"path": p, "size": len(c.encode("utf-8"))}
|
||||
for p, c in sorted(self.endpoint_files.items())]}
|
||||
if path in self.endpoint_files:
|
||||
return {"content": self.endpoint_files[path]}
|
||||
raise RuntimeError("no such file")
|
||||
|
||||
def get_trace_text(self, task_id):
|
||||
return "[ACTION] fake probe\n[OBSERVATION]\n" + self.tasks[task_id].result_text
|
||||
|
||||
@@ -192,6 +213,22 @@ class TestParseHelpers(unittest.TestCase):
|
||||
self.assertIsNone(parse_choice("no choice here"))
|
||||
self.assertEqual(parse_why("<why>because</why>"), "because")
|
||||
self.assertEqual(parse_notes("<notes>did x</notes>"), "did x")
|
||||
|
||||
def test_parse_deliverable_files_lenient(self):
|
||||
text = ("=== DELIVERABLE FILE: a.md ===\nhello\n"
|
||||
"=== END DELIVERABLE FILE ===\n"
|
||||
"=== DELIVERABLE FILE: ../evil ===\nx\n"
|
||||
"=== END DELIVERABLE FILE ===\n"
|
||||
"=== DELIVERABLE FILE: b.md ===\nunterminated\n")
|
||||
self.assertEqual(parse_deliverable_files(text), {"a.md": "hello\n"})
|
||||
many = "".join(
|
||||
f"=== DELIVERABLE FILE: f{i}.md ===\nx\n"
|
||||
"=== END DELIVERABLE FILE ===\n" for i in range(10))
|
||||
self.assertEqual(len(parse_deliverable_files(many)), 8)
|
||||
big = ("=== DELIVERABLE FILE: big.md ===\n" + "x" * 70000 + "\n"
|
||||
"=== END DELIVERABLE FILE ===\n")
|
||||
content = parse_deliverable_files(big)["big.md"]
|
||||
self.assertTrue(content.endswith("...[truncated]"))
|
||||
exp = parse_experience(
|
||||
"<experience>\ncue: c\nlesson: l\nevidence: e\nconfidence: high\n</experience>")
|
||||
self.assertEqual((exp["cue"], exp["lesson"], exp["evidence"]), ("c", "l", "e"))
|
||||
@@ -238,6 +275,109 @@ class TestFullRunViaTasks(unittest.TestCase):
|
||||
self.assertIsNotNone(result.library_entry)
|
||||
self.assertTrue((root / "library" / "accounting" / "INDEX.md").is_file())
|
||||
|
||||
def _models_by_role(self, client):
|
||||
roles = {}
|
||||
for key, tid in client.by_key.items():
|
||||
for role in ("probe", "edit", "select", "reflect"):
|
||||
if f"-{role}-" in key or key.endswith(f"-{role}"):
|
||||
roles.setdefault(role, set()).add(client.task_models[tid])
|
||||
return roles
|
||||
|
||||
def test_role_models_land_on_each_role(self):
|
||||
client = FakeTaskClient()
|
||||
tmp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(tmp.cleanup)
|
||||
root = Path(tmp.name)
|
||||
result = run_improvement_via_tasks(
|
||||
client, project_id="p1", task_id="ledger-1",
|
||||
prompt="Sum every signed amount in ledger.csv. Return Total: with two decimals.",
|
||||
base_harness=dict(BASE_TREE), run_tag="roles1",
|
||||
work_dir=root / "work", domain="accounting",
|
||||
source_files={"ledger.csv": LEDGER},
|
||||
k=2, mode="parallel", library_root=root / "library",
|
||||
model="shared-model",
|
||||
role_models={"probe": "probe-model", "editor": "editor-model",
|
||||
"selector": "selector-model",
|
||||
"reflector": "reflector-model"})
|
||||
self.assertTrue(result.ok)
|
||||
roles = self._models_by_role(client)
|
||||
self.assertEqual(roles["probe"], {"probe-model"})
|
||||
self.assertEqual(roles["edit"], {"editor-model"})
|
||||
self.assertEqual(roles["select"], {"selector-model"})
|
||||
self.assertEqual(roles["reflect"], {"reflector-model"})
|
||||
|
||||
def test_role_model_falls_back_to_shared_then_default(self):
|
||||
client = FakeTaskClient()
|
||||
tmp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(tmp.cleanup)
|
||||
root = Path(tmp.name)
|
||||
result = run_improvement_via_tasks(
|
||||
client, project_id="p1", task_id="ledger-1",
|
||||
prompt="Sum every signed amount in ledger.csv. Return Total: with two decimals.",
|
||||
base_harness=dict(BASE_TREE), run_tag="roles2",
|
||||
work_dir=root / "work", domain="accounting",
|
||||
source_files={"ledger.csv": LEDGER},
|
||||
k=1, mode="parallel", library_root=root / "library",
|
||||
model="shared-model",
|
||||
role_models={"editor": "editor-model"})
|
||||
self.assertTrue(result.ok)
|
||||
roles = self._models_by_role(client)
|
||||
self.assertEqual(roles["probe"], {"shared-model"})
|
||||
self.assertEqual(roles["edit"], {"editor-model"})
|
||||
self.assertEqual(roles["reflect"], {"shared-model"})
|
||||
# 两者都空时不传模型(服务端默认),假客户端记为 None。
|
||||
client2 = FakeTaskClient()
|
||||
result2, _, _ = self._run(client2, run_tag="roles3")
|
||||
self.assertTrue(result2.ok)
|
||||
self.assertEqual(set(client2.task_models.values()), {None})
|
||||
|
||||
def test_probe_deliverable_files_from_endpoint(self):
|
||||
client = FakeTaskClient(endpoint_files={
|
||||
"report.md": "# R\n",
|
||||
"ledger.csv": "amount\n999.00\n",
|
||||
"AGENTS.md": BASE_TREE["AGENTS.md"],
|
||||
})
|
||||
result, _, root = self._run(client, run_tag="assets1")
|
||||
self.assertTrue(result.ok)
|
||||
agg = result.mode_result.rollouts["agg1"]
|
||||
# 新增文件与改动过的 seed 文件进产物;内容与 seed 一致的不进。
|
||||
self.assertEqual(agg.deliverable_files,
|
||||
{"report.md": "# R\n",
|
||||
"ledger.csv": "amount\n999.00\n"})
|
||||
found = [p for p in root.rglob("report.md")
|
||||
if "deliverables/agg1/files" in str(p)]
|
||||
self.assertTrue(found)
|
||||
self.assertEqual(found[0].read_text(encoding="utf-8"), "# R\n")
|
||||
|
||||
def test_probe_deliverable_files_from_reply_blocks(self):
|
||||
block = ("\n=== DELIVERABLE FILE: page.html ===\n<h1>hi</h1>\n"
|
||||
"=== END DELIVERABLE FILE ===\n")
|
||||
client = FakeTaskClient(endpoint_error=True, probe_extra=block)
|
||||
result, _, _ = self._run(client, run_tag="assets2")
|
||||
self.assertTrue(result.ok)
|
||||
self.assertEqual(
|
||||
result.mode_result.rollouts["agg1"].deliverable_files,
|
||||
{"page.html": "<h1>hi</h1>\n"})
|
||||
|
||||
def test_probe_deliverable_files_endpoint_wins_on_collision(self):
|
||||
block = ("\n=== DELIVERABLE FILE: page.html ===\nfrom-reply\n"
|
||||
"=== END DELIVERABLE FILE ===\n")
|
||||
client = FakeTaskClient(
|
||||
endpoint_files={"page.html": "from-endpoint\n"},
|
||||
probe_extra=block)
|
||||
result, _, _ = self._run(client, run_tag="assets3")
|
||||
self.assertTrue(result.ok)
|
||||
self.assertEqual(
|
||||
result.mode_result.rollouts["agg1"].deliverable_files,
|
||||
{"page.html": "from-endpoint\n"})
|
||||
|
||||
def test_probe_deliverable_files_endpoint_error_is_not_fatal(self):
|
||||
client = FakeTaskClient(endpoint_error=True)
|
||||
result, _, _ = self._run(client, run_tag="assets4")
|
||||
self.assertTrue(result.ok)
|
||||
self.assertEqual(
|
||||
result.mode_result.rollouts["agg1"].deliverable_files, {})
|
||||
|
||||
def test_same_run_tag_replays_without_new_tasks(self):
|
||||
client = FakeTaskClient()
|
||||
result1, _, _ = self._run(client, work_sub="work1")
|
||||
@@ -563,5 +703,56 @@ class TestHttpClientRetry(unittest.TestCase):
|
||||
self.assertEqual(counts["n"], 1)
|
||||
|
||||
|
||||
class TestSweep(unittest.TestCase):
|
||||
def _spec(self, tag, **over):
|
||||
spec = {
|
||||
"run_tag": tag, "task_id": "ledger-1",
|
||||
"prompt": "Sum every signed amount in ledger.csv. Return Total: with two decimals.",
|
||||
"base_harness": dict(BASE_TREE),
|
||||
"source_files": {"ledger.csv": LEDGER},
|
||||
"k": 1, "mode": "parallel", "domain": "accounting",
|
||||
}
|
||||
spec.update(over)
|
||||
return spec
|
||||
|
||||
def test_sweep_isolates_failures_and_shares_library(self):
|
||||
client = FakeTaskClient(fail_tags=("bad1",))
|
||||
tmp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(tmp.cleanup)
|
||||
root = Path(tmp.name)
|
||||
report = run_sweep(
|
||||
client, project_id="p1",
|
||||
runs=[self._spec("good1"), self._spec("bad1"),
|
||||
self._spec("badk", k=99), self._spec("good2")],
|
||||
work_root=root / "sweep", library_root=root / "library")
|
||||
by_tag = {e["run_tag"]: e for e in report["runs"]}
|
||||
self.assertTrue(by_tag["good1"]["ok"])
|
||||
self.assertEqual(by_tag["good1"]["chosen_label"], "agg1")
|
||||
# bad1:全部任务注入失败 → 轮次 ok=False,但不中断后续轮。
|
||||
self.assertFalse(by_tag["bad1"]["ok"])
|
||||
# badk:k 越界在方法层抛错 → 记入 error,同样不中断。
|
||||
self.assertFalse(by_tag["badk"]["ok"])
|
||||
self.assertIn("ValueError", by_tag["badk"]["error"])
|
||||
self.assertTrue(by_tag["good2"]["ok"])
|
||||
self.assertEqual(report["totals"]["n_runs"], 4)
|
||||
self.assertEqual(report["totals"]["n_ok"], 2)
|
||||
# 共享库:两轮好结果都入库,同一 INDEX 可见。
|
||||
self.assertIsNotNone(by_tag["good1"]["library_entry"])
|
||||
self.assertIsNotNone(by_tag["good2"]["library_entry"])
|
||||
self.assertTrue((root / "library" / "accounting" / "INDEX.md").is_file())
|
||||
|
||||
def test_sweep_requires_run_tag(self):
|
||||
client = FakeTaskClient()
|
||||
tmp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(tmp.cleanup)
|
||||
root = Path(tmp.name)
|
||||
spec = self._spec("good1")
|
||||
del spec["run_tag"]
|
||||
report = run_sweep(client, project_id="p1", runs=[spec],
|
||||
work_root=root / "sweep")
|
||||
self.assertFalse(report["runs"][0]["ok"])
|
||||
self.assertIn("run_tag", report["runs"][0]["error"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,165 @@
|
||||
"""GET /tasks/{id}/files 端点测试:本地 worktree 文件读取。
|
||||
|
||||
只用标准库 http.client,不跑真实模型,不联网。
|
||||
任务行直接插库(不走 POST /tasks,避免 worker 派发),
|
||||
worktree 用 server.host_worktrees_dir 注入到临时目录。
|
||||
"""
|
||||
|
||||
import http.client
|
||||
import json
|
||||
import os
|
||||
import sqlite3
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from urllib.parse import quote
|
||||
|
||||
try:
|
||||
from cloud.auth import AuthStore
|
||||
from cloud.server import create_server
|
||||
except ImportError:
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||||
from cloud.auth import AuthStore
|
||||
from cloud.server import create_server
|
||||
|
||||
|
||||
class TaskFilesTestCase(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.db_path = os.path.join(self.tmp.name, "control.db")
|
||||
self.auth = AuthStore(self.db_path)
|
||||
self.owner_id, self.owner_key = self.auth.create_user("owner")
|
||||
self.viewer_id, self.viewer_key = self.auth.create_user("viewer")
|
||||
self.outsider_id, self.outsider_key = self.auth.create_user("outsider")
|
||||
self.server = create_server(self.db_path, "127.0.0.1", 0)
|
||||
self.worktrees = os.path.join(self.tmp.name, "host-worktrees")
|
||||
os.makedirs(self.worktrees)
|
||||
self.server.host_worktrees_dir = self.worktrees
|
||||
self.port = self.server.server_address[1]
|
||||
self.thread = threading.Thread(
|
||||
target=self.server.serve_forever, daemon=True)
|
||||
self.thread.start()
|
||||
status, self.project = self._req(
|
||||
"POST", "/projects", {"name": "demo"}, self.owner_key)
|
||||
self.assertEqual(status, 201)
|
||||
self.auth.set_member(
|
||||
self.project["id"], self.owner_id, self.viewer_id, "viewer")
|
||||
self._task("task-files-1", executor=None)
|
||||
wt = os.path.join(self.worktrees, "task-files-1")
|
||||
os.makedirs(os.path.join(wt, "sub"))
|
||||
Path(wt, "report.md").write_text("# Report\n", encoding="utf-8")
|
||||
Path(wt, "sub", "page.html").write_text("<h1>hi</h1>\n",
|
||||
encoding="utf-8")
|
||||
# 非法 UTF-8 字节(\x00 是合法 UTF-8,不能用来测二进制拒绝)。
|
||||
Path(wt, "blob.bin").write_bytes(b"\x89PNG\r\n\x1a\n\xff\xfe")
|
||||
Path(wt, "big.txt").write_text("x" * (256 * 1024 + 1),
|
||||
encoding="utf-8")
|
||||
os.symlink(os.path.join(self.tmp.name, "outside.txt"),
|
||||
os.path.join(wt, "escape"))
|
||||
Path(self.tmp.name, "outside.txt").write_text("secret\n",
|
||||
encoding="utf-8")
|
||||
self._task("task-docker-1", executor="docker")
|
||||
self._task("task-nowt-1", executor=None)
|
||||
|
||||
def tearDown(self):
|
||||
self.server.shutdown()
|
||||
self.server.server_close()
|
||||
self.thread.join(timeout=5)
|
||||
try:
|
||||
self.auth.conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
self.tmp.cleanup()
|
||||
|
||||
def _task(self, task_id, executor):
|
||||
conn = sqlite3.connect(self.db_path)
|
||||
try:
|
||||
conn.execute(
|
||||
"INSERT INTO tasks(id, project_id, prompt, status, executor)"
|
||||
" VALUES(?, ?, ?, ?, ?)",
|
||||
(task_id, self.project["id"], "do it", "completed", executor))
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def _req(self, method, path, body=None, key=None):
|
||||
conn = http.client.HTTPConnection("127.0.0.1", self.port, timeout=10)
|
||||
hdrs = {}
|
||||
payload = None
|
||||
if body is not None:
|
||||
payload = json.dumps(body).encode("utf-8")
|
||||
hdrs["Content-Type"] = "application/json"
|
||||
if key is not None:
|
||||
hdrs["X-API-Key"] = key
|
||||
try:
|
||||
conn.request(method, path, body=payload, headers=hdrs)
|
||||
resp = conn.getresponse()
|
||||
data = resp.read()
|
||||
finally:
|
||||
conn.close()
|
||||
try:
|
||||
parsed = json.loads(data.decode("utf-8")) if data else None
|
||||
except ValueError:
|
||||
parsed = None
|
||||
return resp.status, parsed
|
||||
|
||||
def _files(self, task_id, path=None, key=None):
|
||||
url = f"/tasks/{task_id}/files"
|
||||
if path is not None:
|
||||
url += "?path=" + quote(path)
|
||||
return self._req("GET", url, key=self.owner_key if key is None else key)
|
||||
|
||||
def test_list_top_level_recursive(self):
|
||||
status, body = self._files("task-files-1")
|
||||
self.assertEqual(status, 200, body)
|
||||
paths = sorted(e["path"] for e in body["entries"])
|
||||
self.assertIn("report.md", paths)
|
||||
self.assertIn("sub/page.html", paths)
|
||||
self.assertNotIn("escape", paths) # 符号链接不跟进列表
|
||||
|
||||
def test_read_file_and_subdir_listing(self):
|
||||
status, body = self._files("task-files-1", "report.md")
|
||||
self.assertEqual(status, 200, body)
|
||||
self.assertEqual(body["content"], "# Report\n")
|
||||
status, body = self._files("task-files-1", "sub")
|
||||
self.assertEqual(status, 200, body)
|
||||
self.assertEqual([e["path"] for e in body["entries"]],
|
||||
["sub/page.html"])
|
||||
|
||||
def test_path_escape_blocked(self):
|
||||
for bad in ("../outside.txt", "/etc/passwd", "escape", "sub/../../x"):
|
||||
status, body = self._files("task-files-1", bad)
|
||||
self.assertIn(status, (400, 404), (bad, body))
|
||||
# escape 是指到 worktree 外的符号链接:必须 400,不是读到 secret。
|
||||
status, body = self._files("task-files-1", "escape")
|
||||
self.assertEqual(status, 400, body)
|
||||
|
||||
def test_missing_file_and_missing_worktree(self):
|
||||
status, _ = self._files("task-files-1", "nope.md")
|
||||
self.assertEqual(status, 404)
|
||||
status, _ = self._files("task-nowt-1")
|
||||
self.assertEqual(status, 404)
|
||||
status, _ = self._files("task-unknown")
|
||||
self.assertEqual(status, 404)
|
||||
|
||||
def test_binary_and_oversized(self):
|
||||
status, body = self._files("task-files-1", "blob.bin")
|
||||
self.assertEqual(status, 409, body)
|
||||
status, body = self._files("task-files-1", "big.txt")
|
||||
self.assertEqual(status, 413, body)
|
||||
|
||||
def test_docker_task_409_and_auth(self):
|
||||
status, body = self._files("task-docker-1")
|
||||
self.assertEqual(status, 409, body)
|
||||
status, _ = self._req("GET", "/tasks/task-files-1/files")
|
||||
self.assertEqual(status, 401)
|
||||
status, _ = self._files("task-files-1", key=self.outsider_key)
|
||||
self.assertEqual(status, 403)
|
||||
status, _ = self._files("task-files-1", key=self.viewer_key)
|
||||
self.assertEqual(status, 200)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
+3
-3
@@ -34,9 +34,9 @@ benchmark 上的,不是本项目的承诺——本项目没有 benchmark,接
|
||||
| 经验库(分领域、整树 + EXPERIENCE.md + INDEX.md) | 照搬存储格式,改写生效语义:入库 ≠ 自动生效(见下) |
|
||||
| Ledger(会话/探针账本)与成本 | 改写:成本汇总进 `ModeResult.total_cost_usd`,由调用方接现有 `cloud/cost.py` / 租户配额计费;另有 `budget_usd` 超支门与 `max_sessions` 会话数门(计探针 + 阶段尝试会话;edit 内 fmt2 纠正与选择/反思会话不计入):任一超支后不再调用编辑器/探针,剩余阶段回落,且该轮不入库;上游的 R=4k/P=3k 事后强校验未引(本项目结构与上游计数不完全同构,门控代替断言) |
|
||||
| run_tag 形状与建任务前预检 | 对照补齐:`validate_run_tag`(字符集、首字符不为 -、不以 s 数字段开头)与 `preflight_checks`(base 目录与 AGENTS.md、source 目录、提示、tag、控制面 health 探针);CLI `--check` 只预检不建任务 |
|
||||
| 分角色模型(SOLVER / IMPROVER / JUDGE) | 未引:全部角色共用调用方透传的同一个 model;需用户定各角色模型才有意义 |
|
||||
| 产物资产(deliverable_files / deliverable_assets / images)与选择 assets note | 未引:探针交付物目前只有最终回复文本;需新增 worktree 文件读取端点后与该端点一起做 |
|
||||
| sweep 层(多任务批量、跨任务 library restore、dead 隔离、report 汇总) | 未引:单任务产品形态用不上,批量改进需求出现再做 |
|
||||
| 分角色模型(SOLVER / IMPROVER / JUDGE) | 已引(机制):`run_improvement_via_tasks` 的 `role_models`(probe / editor / selector / reflector)+ CLI `--model-<角色>`;非空值覆盖该角色,缺键回落共享 `--model`,两者都空走服务端默认——默认行为不变,模型选择权仍在调用方(有价模型产生真实成本) |
|
||||
| 产物资产(deliverable_files / deliverable_assets / images)与选择 assets note | 已引(两层):①服务端 `GET /tasks/{id}/files`(本地任务 worktree 读取,路径安全解析、文本 ≤256KB、列表 ≤200 条;Docker/跨机任务 409);②探针回复里的 `DELIVERABLE FILE` 块(所有执行器可用,端点结果优先、坏块丢弃不炸轮次)。产物进 `Rollout.deliverable_files`,并写入选择/编辑/反思上下文的 `deliverables/<label>/files/`。边界:租户 Docker 下 `/app/work` 是整项目共享目录,按任务归属读文件会错指,故端点对 Docker 任务只 409、不给错数据;此时只有 agent 自报层可用。images(二进制产物)未引:端点只读 UTF-8 文本 |
|
||||
| sweep 层(多任务批量、跨任务 library restore、dead 隔离、report 汇总) | 已引:`run_sweep` + CLI `examples/harness2_sweep.py`(共享 library_root、单轮失败隔离、sweep-report.json 汇总;manifest 预检不过不建任务,默认只打印计划,`--yes` 才真跑) |
|
||||
| 提示词 | 改写短版(中文项目语境下的英文任务提示,结构同上游,措辞自写),不抄上游长提示 |
|
||||
| Benchmark 适配器(JobBench / LAB / WorkBuddy)、评分器、rubric | 不引。任务就是一个普通 codex 任务,delivered 由 adapter 定义(如产物存在且非空) |
|
||||
| Proposer 通道(Claude CLI / OpenCode)、Vertex 配置 | 不引。编辑器/选择器/反思器是调用方注入的协议,真实接线时是 codex 任务(见下) |
|
||||
|
||||
@@ -69,6 +69,10 @@ def main():
|
||||
ap.add_argument("--output", default="")
|
||||
ap.add_argument("--library-root", default="")
|
||||
ap.add_argument("--model", default=None)
|
||||
ap.add_argument("--model-probe", default=None)
|
||||
ap.add_argument("--model-editor", default=None)
|
||||
ap.add_argument("--model-selector", default=None)
|
||||
ap.add_argument("--model-reflector", default=None)
|
||||
ap.add_argument("--effort", default=None)
|
||||
ap.add_argument("--approval", default=None)
|
||||
ap.add_argument("--environment-id", default=None)
|
||||
@@ -148,7 +152,11 @@ def main():
|
||||
k=args.k, mode=args.mode,
|
||||
library_root=Path(args.library_root) if args.library_root else None,
|
||||
budget_usd=args.budget, max_sessions=args.max_sessions,
|
||||
model=args.model, effort=args.effort,
|
||||
model=args.model,
|
||||
role_models={"probe": args.model_probe, "editor": args.model_editor,
|
||||
"selector": args.model_selector,
|
||||
"reflector": args.model_reflector},
|
||||
effort=args.effort,
|
||||
approval=args.approval, environment_id=args.environment_id,
|
||||
executor=args.executor, no_time_limit=args.no_time_limit,
|
||||
timeout_seconds=args.timeout)
|
||||
|
||||
@@ -0,0 +1,176 @@
|
||||
"""多轮 Harness² 改进批量跑(sweep,接线层 CLI)。
|
||||
|
||||
读一个 JSON manifest,顺序跑多轮 run_improvement_via_tasks:所有轮
|
||||
共享同一个 library_root(已入库的经验会被后续轮读到,即上游 sweep
|
||||
的 library restore 语义);单轮失败只记入报告、不中断后续轮
|
||||
(dead 隔离);最后写 sweep-report.json 并汇总打印。
|
||||
|
||||
这会创建真实任务、花真实模型配额——所以默认只打印计划,不建
|
||||
任务:先跑一遍不带 --yes 看计划,确认无误后,再带 --yes 重跑。
|
||||
密钥不走命令行参数(ps 可见):用环境变量 CODEX_API_KEY。
|
||||
|
||||
manifest 形状:
|
||||
{
|
||||
"runs": [
|
||||
{"run_tag": "sw1", "task_id": "ledger-1",
|
||||
"prompt": "Sum every signed amount in ledger.csv.",
|
||||
"base_harness_dir": "./base-harness", "source_dir": "./task-source",
|
||||
"domain": "accounting", "k": 1, "mode": "parallel",
|
||||
"budget": 5, "max_sessions": null, "model": null,
|
||||
"role_models": null, "effort": null, "approval": null,
|
||||
"environment_id": null, "executor": null,
|
||||
"no_time_limit": false, "timeout": 3600}
|
||||
]
|
||||
}
|
||||
prompt 也可用 prompt_file(相对 manifest 所在目录);base_harness_dir
|
||||
与 source_dir 同样相对 manifest 目录解析(绝对路径原样用)。
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
from cloud.harness2 import read_tree # noqa: E402
|
||||
from cloud.harness2_tasks import ( # noqa: E402
|
||||
HttpTaskClient, preflight_checks, run_sweep, validate_run_tag,
|
||||
)
|
||||
|
||||
|
||||
def _flatten(root: Path) -> dict[str, bytes]:
|
||||
out: dict[str, bytes] = {}
|
||||
if not root or not root.is_dir():
|
||||
return out
|
||||
for p in sorted(root.rglob("*")):
|
||||
if p.is_file() and not p.is_symlink() and "__pycache__" not in p.parts:
|
||||
out[str(p.relative_to(root))] = p.read_bytes()
|
||||
return out
|
||||
|
||||
|
||||
def _resolve(manifest_dir: Path, value):
|
||||
if not value:
|
||||
return value
|
||||
p = Path(value)
|
||||
return p if p.is_absolute() else manifest_dir / p
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--manifest", required=True, help="JSON manifest 路径")
|
||||
ap.add_argument("--base-url", required=True, help="控制面地址")
|
||||
ap.add_argument("--project-id", required=True)
|
||||
ap.add_argument("--output", default="",
|
||||
help="sweep 工作根目录(默认临时目录)")
|
||||
ap.add_argument("--library-root", default="",
|
||||
help="共享经验库根目录(空 = 各轮不入库)")
|
||||
ap.add_argument("--yes", action="store_true", help="确认创建真实任务并消费配额")
|
||||
args = ap.parse_args()
|
||||
|
||||
manifest_path = Path(args.manifest)
|
||||
try:
|
||||
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
except (OSError, ValueError) as exc:
|
||||
print(f"manifest 读不到或不是合法 JSON:{exc}", file=sys.stderr)
|
||||
return 2
|
||||
runs = manifest.get("runs")
|
||||
if not isinstance(runs, list) or not runs:
|
||||
print("manifest 里 runs 必须是非空列表", file=sys.stderr)
|
||||
return 2
|
||||
manifest_dir = manifest_path.resolve().parent
|
||||
|
||||
specs = []
|
||||
seen_tags = set()
|
||||
for i, raw in enumerate(runs):
|
||||
if not isinstance(raw, dict):
|
||||
print(f"runs[{i}] 不是对象", file=sys.stderr)
|
||||
return 2
|
||||
tag = raw.get("run_tag", "")
|
||||
try:
|
||||
if not tag:
|
||||
raise ValueError("每轮必须给 run_tag")
|
||||
validate_run_tag(tag)
|
||||
except ValueError as exc:
|
||||
print(f"runs[{i}] run_tag 非法:{exc}", file=sys.stderr)
|
||||
return 2
|
||||
if tag in seen_tags:
|
||||
print(f"runs[{i}] run_tag 重复:{tag}", file=sys.stderr)
|
||||
return 2
|
||||
seen_tags.add(tag)
|
||||
prompt = raw.get("prompt", "")
|
||||
if raw.get("prompt_file"):
|
||||
try:
|
||||
prompt = _resolve(manifest_dir, raw["prompt_file"]).read_text(
|
||||
encoding="utf-8")
|
||||
except OSError as exc:
|
||||
print(f"runs[{i}] prompt_file 读不到:{exc}", file=sys.stderr)
|
||||
return 2
|
||||
base_dir = _resolve(manifest_dir, raw.get("base_harness_dir", ""))
|
||||
errors, warnings = preflight_checks(
|
||||
base_harness_dir=base_dir,
|
||||
source_dir=_resolve(manifest_dir, raw.get("source_dir")) or None,
|
||||
prompt=prompt, run_tag=tag, base_url=args.base_url)
|
||||
for e in errors:
|
||||
print(f"FAIL runs[{i}] {tag}: {e}", file=sys.stderr)
|
||||
for w in warnings:
|
||||
print(f"WARN runs[{i}] {tag}: {w}")
|
||||
if errors:
|
||||
return 2
|
||||
source_dir = _resolve(manifest_dir, raw.get("source_dir"))
|
||||
specs.append({
|
||||
"run_tag": tag,
|
||||
"task_id": raw.get("task_id", f"h2-task-{tag}"),
|
||||
"prompt": prompt,
|
||||
"base_harness": read_tree(base_dir),
|
||||
"source_files": _flatten(source_dir) if source_dir else None,
|
||||
"domain": raw.get("domain", "general"),
|
||||
"k": raw.get("k", 1),
|
||||
"mode": raw.get("mode", "parallel"),
|
||||
"budget_usd": raw.get("budget"),
|
||||
"max_sessions": raw.get("max_sessions"),
|
||||
"model": raw.get("model"),
|
||||
"role_models": raw.get("role_models"),
|
||||
"effort": raw.get("effort"),
|
||||
"approval": raw.get("approval"),
|
||||
"environment_id": raw.get("environment_id"),
|
||||
"executor": raw.get("executor"),
|
||||
"no_time_limit": bool(raw.get("no_time_limit", False)),
|
||||
"timeout_seconds": raw.get("timeout", 3600),
|
||||
})
|
||||
|
||||
print(f"计划:项目 {args.project_id},共 {len(specs)} 轮,"
|
||||
f"tags={sorted(seen_tags)},"
|
||||
f"共享 library={'是' if args.library_root else '否(不入库)'}")
|
||||
if not args.yes:
|
||||
print("未带 --yes:只打印计划,不创建任何任务。确认无误后加 --yes 重跑。")
|
||||
return 0
|
||||
api_key = os.environ.get("CODEX_API_KEY")
|
||||
if not api_key:
|
||||
print("缺少 CODEX_API_KEY 环境变量(密钥不接受命令行参数)", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
work_root = Path(args.output) if args.output else Path(
|
||||
tempfile.mkdtemp(prefix="h2sweep-"))
|
||||
client = HttpTaskClient(args.base_url, api_key)
|
||||
report = run_sweep(
|
||||
client, project_id=args.project_id, runs=specs, work_root=work_root,
|
||||
library_root=Path(args.library_root) if args.library_root else None)
|
||||
report_path = work_root / "sweep-report.json"
|
||||
report_path.write_text(json.dumps(report, ensure_ascii=False, indent=2),
|
||||
encoding="utf-8")
|
||||
for entry in report["runs"]:
|
||||
status = "ok" if entry["ok"] else f"FAIL {entry['error']}"
|
||||
print(f"{entry['run_tag']}: {status} "
|
||||
f"chosen={entry['chosen_label']} "
|
||||
f"cost={entry['total_cost_usd']}")
|
||||
totals = report["totals"]
|
||||
print(f"汇总:{totals['n_ok']}/{totals['n_runs']} 轮成功,"
|
||||
f"总成本 {totals['total_cost_usd']},报告 {report_path}")
|
||||
return 0 if totals["n_ok"] == totals["n_runs"] else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in new issue
Block a user