harness2: per-role models, sweep batch, task file assets endpoint

- Role models: run_improvement_via_tasks gains role_models
  (probe/editor/selector/reflector) plus --model-<role> CLI flags;
  missing/empty falls back to shared --model, both empty means
  server default, so default behavior is unchanged.
- Sweep: run_sweep + examples/harness2_sweep.py manifest CLI with
  shared library_root (library restore), per-run failure isolation
  and sweep-report.json summary; preflight gates task creation.
- Deliverable assets: GET /tasks/{id}/files for local-executor
  tasks (safe path resolution, UTF-8 text <=256KB, listing <=200;
  docker/cloud/cross-host 409 with explicit reason, since tenant
  Docker shares one /app/work per project and per-task attribution
  would misattribute); probe rollout deliverable_files from the
  endpoint plus DELIVERABLE FILE reply blocks (endpoint wins on
  collision, bad blocks dropped), surfaced in
  deliverables/<label>/files/ context and full_result summary.
- Docs: harness2.md comparison table updated for all three items.
This commit is contained in:
host committed 2026-10-29 16:32:58 +08:00
1 parent 9e444eca38
commit c20192bce8
8 files changed
+856 -28

No files matched your search

+58 -1
View File
@@ -241,6 +241,56 @@ def parse_harness_files(text: str) -> dict[str, str]:
return out return out
_DELIVERABLE_FILE_START_RX = re.compile(r"^=== DELIVERABLE FILE:\s*(.+?)\s*===$")
_DELIVERABLE_FILE_END = "=== END DELIVERABLE FILE ==="
_DELIVERABLE_FILE_MAX = 8
_DELIVERABLE_FILE_MAX_CHARS = 65536
def _is_safe_deliverable_path(path: str) -> bool:
if not path or path.startswith("/") or "\x00" in path:
return False
return all(p not in ("", ".", "..") for p in path.split("/"))
def parse_deliverable_files(text: str) -> dict[str, str]:
"""解析探针回复里的产物文件块,返回 {相对路径: 内容}。
与编辑文件块的严格语义不同:这里故意宽容——坏块(未闭合、
非法路径、超量、超长)只丢弃该块、不抛错,因为产物文件是
交付文本之外的 enrichment,绝不能因为一块坏内容毁掉整轮
交付物。超长内容截断并打截断标记,让评审知道它不完整;
超过 8 块时只取前 8 块。同一路径出现两次时后一块覆盖前一块。
"""
out: dict[str, str] = {}
cur_path: str | None = None
cur_lines: list[str] = []
for line in (text or "").splitlines():
m = _DELIVERABLE_FILE_START_RX.match(line.strip())
if m:
if cur_path is not None:
cur_path = None
cur_lines = []
path = m.group(1).strip()
if _is_safe_deliverable_path(path) and len(out) < _DELIVERABLE_FILE_MAX:
cur_path = path
cur_lines = []
continue
if line.strip() == _DELIVERABLE_FILE_END:
if cur_path is not None:
content = "\n".join(cur_lines) + ("\n" if cur_lines else "")
if len(content) > _DELIVERABLE_FILE_MAX_CHARS:
content = (content[:_DELIVERABLE_FILE_MAX_CHARS]
+ "\n...[truncated]")
out[cur_path] = content
cur_path = None
cur_lines = []
continue
if cur_path is not None:
cur_lines.append(line)
return out
MAX_K = 3 MAX_K = 3
STAGE_ATTEMPTS = 2 # 提议被拒后给一次按反馈修正的机会,再拒就回落 STAGE_ATTEMPTS = 2 # 提议被拒后给一次按反馈修正的机会,再拒就回落
@@ -490,6 +540,7 @@ class Rollout:
trajectory: str = "" trajectory: str = ""
cost_usd: float = 0.0 cost_usd: float = 0.0
error: str = "" error: str = ""
deliverable_files: dict[str, str] = field(default_factory=dict)
@dataclass @dataclass
@@ -589,7 +640,12 @@ def build_context(
(dd / f"{r.label}.md").write_text( (dd / f"{r.label}.md").write_text(
r.deliverable_text or "(no deliverable)", encoding="utf-8" r.deliverable_text or "(no deliverable)", encoding="utf-8"
) )
lines.append("context/deliverables/ what each rollout produced") for name, content in (r.deliverable_files or {}).items():
dest = dd / r.label / "files" / name
dest.parent.mkdir(parents=True, exist_ok=True)
dest.write_text(content, encoding="utf-8")
lines.append("context/deliverables/ what each rollout produced"
" (+ <label>/files/ when it produced files)")
write_tree(harness, root / "harness") write_tree(harness, root / "harness")
lines.append("context/harness/ the harness tree you may edit, and ONLY this") lines.append("context/harness/ the harness tree you may edit, and ONLY this")
if library_dir and Path(library_dir).is_dir(): if library_dir and Path(library_dir).is_dir():
@@ -842,6 +898,7 @@ class ModeResult:
"delivered": r.delivered, "delivered": r.delivered,
"error": (r.error or "")[:400], "error": (r.error or "")[:400],
"deliverable_text": (r.deliverable_text or "")[:500], "deliverable_text": (r.deliverable_text or "")[:500],
"deliverable_files": sorted(r.deliverable_files or {}),
} }
for lbl, r in self.rollouts.items() for lbl, r in self.rollouts.items()
}, },
+145 -13
View File
@@ -1,6 +1,6 @@
"""Harness² 接线层:把 cloud/harness2.py 的三个协议落到真实 codex 任务上。 """Harness² 接线层:把 cloud/harness2.py 的三个协议落到真实 codex 任务上。
只用控制面已有的 API,不新增任何服务端端点: 控制面 API:
POST /uploads(multipart + paths JSON)-> upload_id POST /uploads(multipart + paths JSON)-> upload_id
POST /tasks(upload_id 绑定 + idempotency_key 幂等)-> 任务排队 POST /tasks(upload_id 绑定 + idempotency_key 幂等)-> 任务排队
GET /tasks/{id} 轮询终态(completed / failed / interrupted / GET /tasks/{id} 轮询终态(completed / failed / interrupted /
@@ -8,6 +8,8 @@
GET /tasks/{id}/trace 轨迹(探针的 trajectory 来源) GET /tasks/{id}/trace 轨迹(探针的 trajectory 来源)
GET /tasks/{id}/cost 成本(usd_micros / 1e6;无价 token 不计入 USD, GET /tasks/{id}/cost 成本(usd_micros / 1e6;无价 token 不计入 USD,
这是现有 cost 模型的边界,不是本层漏算) 这是现有 cost 模型的边界,不是本层漏算)
GET /tasks/{id}/files 读本地任务 worktree 文件(产物资产端点;
Docker/跨机任务回 409,见下)
POST /tasks/{id}/cancel 超时取消,不留孤儿任务继续烧钱 POST /tasks/{id}/cancel 超时取消,不留孤儿任务继续烧钱
认证:X-API-Key 请求头;密钥只由调用方持有,本模块不读、不存、 认证:X-API-Key 请求头;密钥只由调用方持有,本模块不读、不存、
不打印密钥。 不打印密钥。
@@ -18,12 +20,13 @@
文件会覆盖仓库同名文件——这正是探针要的效果:同一任务在不同 文件会覆盖仓库同名文件——这正是探针要的效果:同一任务在不同
harness 下跑。prompt = 原任务 prompt + 一段交付脚注(见 harness 下跑。prompt = 原任务 prompt + 一段交付脚注(见
_PROBE_DELIVERABLE_FOOTER),因为评审只看最终回复:agent 必须 _PROBE_DELIVERABLE_FOOTER),因为评审只看最终回复:agent 必须
把交付物完整写在最终回复里,不能只说“文件已写好”。这就是本层 把交付物完整写在最终回复里,不能只说“文件已写好”。
最大的适用边界:交付物是最终回复文本的任务;文件型产物(代码 文件型产物走两层(产物资产):①本地任务经 GET /tasks/{id}/files
diff、报告文件)目前只能靠最终回复复述/摘要,worktree 快照 API 读 worktree 新增/改动文件(端点为准);②所有执行器都可用
只给 diff 截断与未跟踪文件列表、不给完整文件内容,跨主机还读 回复里的 DELIVERABLE FILE 块(agent 自报,端点结果优先)。
不到——文件产物评审需要新增文件读取端点,另行评审,不在本层 边界仍在:租户 Docker 下 /app/work 是整项目共享目录,按任务
假装已解决。 归属读文件会错指,端点对 Docker 任务一律 409——这时只有第
②层可用,且内容以 agent 自报为准(可能截断,截断有标记)。
- 编辑(CodexEditor.edit):bundle = context/ 前缀的证据树(task.md / - 编辑(CodexEditor.edit):bundle = context/ 前缀的证据树(task.md /
rollouts / deliverables / harness / library / pool / decisions)。 rollouts / deliverables / harness / library / pool / decisions)。
agent 在 worktree 的 context/harness/ 里改文件,最终回复里用 agent 在 worktree 的 context/harness/ 里改文件,最终回复里用
@@ -65,6 +68,7 @@ import json
import re import re
import time import time
import urllib.error import urllib.error
import urllib.parse
import urllib.request import urllib.request
import uuid import uuid
from dataclasses import dataclass, field from dataclasses import dataclass, field
@@ -77,6 +81,7 @@ from .harness2 import (
Tree, Tree,
improve_task, improve_task,
parse_choice, parse_choice,
parse_deliverable_files,
parse_experience, parse_experience,
parse_harness_files, parse_harness_files,
parse_no_change, parse_no_change,
@@ -185,6 +190,11 @@ write the complete deliverable in your final reply itself — do not only say
"done" or "the file is written". If the task asks for a file, also write the "done" or "the file is written". If the task asks for a file, also write the
file in the worktree, but the final reply must still contain the full content file in the worktree, but the final reply must still contain the full content
a reviewer needs to judge the work. a reviewer needs to judge the work.
If you created deliverable files (a page, a report, an image manifest),
also paste each file's full content in your final reply as a block:
=== DELIVERABLE FILE: <relative path> ===
<complete file content>
=== END DELIVERABLE FILE ===
""" """
_EDITOR_OUTPUT_FORMAT = """ _EDITOR_OUTPUT_FORMAT = """
@@ -278,6 +288,12 @@ class TaskClient(Protocol):
"""轨迹渲染文本;读不到时返回空串(调用方用最终回复兜底)。""" """轨迹渲染文本;读不到时返回空串(调用方用最终回复兜底)。"""
... ...
def get_task_files(self, task_id: str, path: str = "") -> dict:
"""GET /tasks/{id}/files:不传 path 列 worktree 文件,传 path
读单个文件。服务端对 Docker/跨机任务回 409,调用方须把
409 当成“无文件通道”、不得当成探针失败。"""
...
def get_cost_usd(self, task_id: str) -> float: def get_cost_usd(self, task_id: str) -> float:
"""有价成本 USD;读不到时返回 0.0,不炸轮次。""" """有价成本 USD;读不到时返回 0.0,不炸轮次。"""
... ...
@@ -452,6 +468,10 @@ class HttpTaskClient:
rendered = "\n".join(lines).strip() rendered = "\n".join(lines).strip()
return rendered[:TRACE_MAX_CHARS] if rendered else "" return rendered[:TRACE_MAX_CHARS] if rendered else ""
def get_task_files(self, task_id: str, path: str = "") -> dict:
q = f"?path={urllib.parse.quote(path)}" if path else ""
return self._request("GET", f"/tasks/{task_id}/files{q}")
def get_cost_usd(self, task_id: str) -> float: def get_cost_usd(self, task_id: str) -> float:
try: try:
obj = self._request("GET", f"/tasks/{task_id}/cost") obj = self._request("GET", f"/tasks/{task_id}/cost")
@@ -596,6 +616,12 @@ class CodexProbeAdapter:
error=f"{type(exc).__name__}: {exc}"[:400]) error=f"{type(exc).__name__}: {exc}"[:400])
state = done.state state = done.state
delivered = state.status == "completed" and bool(state.result_text.strip()) delivered = state.status == "completed" and bool(state.result_text.strip())
deliverable_files = self._collect_deliverable_files(done.task_id, files)
# 回复里的产物文件块是补充通道(所有执行器都可用);
# 端点读到的文件为准,冲突时不覆盖端点结果。
for name, content in parse_deliverable_files(
state.result_text or "").items():
deliverable_files.setdefault(name, content)
return Rollout( return Rollout(
label=label, ok=state.status == "completed", delivered=delivered, label=label, ok=state.status == "completed", delivered=delivered,
deliverable_text=state.result_text, deliverable_text=state.result_text,
@@ -603,8 +629,46 @@ class CodexProbeAdapter:
cost_usd=done.cost_usd, cost_usd=done.cost_usd,
error="" if state.status == "completed" else ( error="" if state.status == "completed" else (
state.error_text or f"task ended as {state.status}")[:400], state.error_text or f"task ended as {state.status}")[:400],
deliverable_files=deliverable_files,
) )
def _collect_deliverable_files(self, task_id: str,
seed_files: dict[str, bytes]) -> dict[str, str]:
"""经 GET /tasks/{id}/files 读探针 worktree 里新增的文件。
取 seed 里没有的新文件,以及内容被改过的 seed 文件(与
seed 字节比对,不一致才算产物);跳过隐藏文件、最多收
8 个、最多读 16 个;端点不可用(Docker/跨机 409、读错)
时返回空 dict——文件产物是 enrichment,绝不能因此炸轮次。
"""
try:
listing = self.client.get_task_files(task_id)
except Exception:
return {}
out: dict[str, str] = {}
reads = 0
for entry in listing.get("entries") or []:
if not isinstance(entry, dict):
continue
path = entry.get("path") or ""
if not path or path.startswith("."):
continue
if len(out) >= 8 or reads >= 16:
break
try:
data = self.client.get_task_files(task_id, path)
except Exception:
continue
reads += 1
content = data.get("content")
if not isinstance(content, str):
continue
seed = seed_files.get(path)
if seed is not None and seed.decode("utf-8", errors="replace") == content:
continue
out[path] = content
return out
@dataclass @dataclass
class CodexEditor: class CodexEditor:
@@ -773,6 +837,7 @@ def run_improvement_via_tasks(
budget_usd: float | None = None, budget_usd: float | None = None,
max_sessions: int | None = None, max_sessions: int | None = None,
model: str | None = None, model: str | None = None,
role_models: dict[str, str | None] | None = None,
effort: str | None = None, effort: str | None = None,
approval: str | None = None, approval: str | None = None,
environment_id: str | None = None, environment_id: str | None = None,
@@ -785,22 +850,37 @@ def run_improvement_via_tasks(
library_root 为空时不建经验库(只跑改进、不入库);非空时按 library_root 为空时不建经验库(只跑改进、不入库);非空时按
domain 建库。auto_apply 恒为 False——采纳的树只入库待评审, domain 建库。auto_apply 恒为 False——采纳的树只入库待评审,
不自动成为后续任务的 base(见 docs/harness2.md 安全一节)。 不自动成为后续任务的 base(见 docs/harness2.md 安全一节)。
分角色模型:role_models 键为 probe / editor / selector /
reflector,非空值覆盖该角色的 model;缺键或空值时回落到
共享 model;两者都空时不传 model(服务端默认模型)。默认
行为与此前完全一致——本参数只提供分流能力,不替用户选
模型(有价模型会产生真实成本,见 docs/harness2.md)。
""" """
from .harness2 import Library from .harness2 import Library
roles = role_models or {}
def _role_model(role: str) -> str | None:
return roles.get(role) or model
adapter = CodexProbeAdapter( adapter = CodexProbeAdapter(
client=client, project_id=project_id, task_id=task_id, prompt=prompt, client=client, project_id=project_id, task_id=task_id, prompt=prompt,
base=base_harness, run_tag=run_tag, base=base_harness, run_tag=run_tag,
source_files=source_files or {}, environment_id=environment_id, source_files=source_files or {}, environment_id=environment_id,
executor=executor, model=model, effort=effort, approval=approval, executor=executor, model=_role_model("probe"), effort=effort,
approval=approval,
no_time_limit=no_time_limit, timeout_seconds=timeout_seconds, no_time_limit=no_time_limit, timeout_seconds=timeout_seconds,
domain=domain) domain=domain)
shared = dict(environment_id=environment_id, executor=executor, model=model, shared = dict(environment_id=environment_id, executor=executor,
effort=effort, approval=approval, no_time_limit=no_time_limit, effort=effort, approval=approval, no_time_limit=no_time_limit,
timeout_seconds=timeout_seconds) timeout_seconds=timeout_seconds)
editor = CodexEditor(client=client, project_id=project_id, run_tag=run_tag, **shared) editor = CodexEditor(client=client, project_id=project_id, run_tag=run_tag,
selector = CodexSelector(client=client, project_id=project_id, run_tag=run_tag, **shared) model=_role_model("editor"), **shared)
reflector = CodexReflector(client=client, project_id=project_id, run_tag=run_tag, **shared) selector = CodexSelector(client=client, project_id=project_id, run_tag=run_tag,
model=_role_model("selector"), **shared)
reflector = CodexReflector(client=client, project_id=project_id, run_tag=run_tag,
model=_role_model("reflector"), **shared)
library = Library(library_root, domain) if library_root is not None else None library = Library(library_root, domain) if library_root is not None else None
validate_run_tag(run_tag) validate_run_tag(run_tag)
return improve_task( return improve_task(
@@ -809,10 +889,62 @@ def run_improvement_via_tasks(
auto_apply=False, budget_usd=budget_usd, max_sessions=max_sessions) auto_apply=False, budget_usd=budget_usd, max_sessions=max_sessions)
def run_sweep(client, *, project_id, runs, work_root, library_root=None):
"""批量跑多轮改进:共享经验库、单轮失败隔离、汇总报告。
runs:每轮一个 dict,键为 run_improvement_via_tasks 的参数,
run_tag 必填;client / project_id / work_dir / library_root
由本函数统管,出现在 spec 里会被忽略(不报错)。每轮
work_dir = work_root/<run_tag>。library_root 非空时所有轮
共享同一库——已入库的经验会被后续轮读到,即上游 sweep 的
library restore 语义;为空时各轮都不入库(仍汇总)。
单轮抛错只记入报告的 error,不中断后续轮(dead 隔离)。
返回报告 dict:runs 逐轮(ok / chosen_label / deliverable
前 500 字 / cost / library_entry / error)+ totals(轮数、
成功轮数、总成本)。
"""
work_root = Path(work_root)
report_runs = []
total_cost = 0.0
n_ok = 0
for spec in runs:
spec = dict(spec)
run_tag = spec.pop("run_tag", "")
for owned in ("client", "project_id", "work_dir", "library_root"):
spec.pop(owned, None)
entry = {"run_tag": run_tag, "ok": False, "chosen_label": None,
"deliverable_text": "", "total_cost_usd": 0.0,
"library_entry": None, "error": None}
try:
if not run_tag:
raise ValueError("sweep 每轮必须给 run_tag(用作 work_dir 子目录)")
validate_run_tag(run_tag)
result = run_improvement_via_tasks(
client, project_id=project_id, run_tag=run_tag,
work_dir=work_root / run_tag,
library_root=library_root, **spec)
entry["ok"] = bool(result.ok)
entry["chosen_label"] = result.selection.chosen_label
chosen = result.mode_result.rollouts.get(result.selection.chosen_label)
if chosen is not None:
entry["deliverable_text"] = (chosen.deliverable_text or "")[:500]
entry["total_cost_usd"] = result.mode_result.total_cost_usd
entry["library_entry"] = result.library_entry
total_cost += result.mode_result.total_cost_usd
n_ok += 1 if result.ok else 0
except Exception as exc:
entry["error"] = f"{type(exc).__name__}: {exc}"[:400]
report_runs.append(entry)
return {"runs": report_runs,
"totals": {"n_runs": len(report_runs), "n_ok": n_ok,
"total_cost_usd": total_cost}}
__all__ = [ __all__ = [
"TERMINAL_STATUSES", "DEFAULT_TASK_TIMEOUT_SECONDS", "DEFAULT_POLL_SECONDS", "TERMINAL_STATUSES", "DEFAULT_TASK_TIMEOUT_SECONDS", "DEFAULT_POLL_SECONDS",
"TRACE_MAX_CHARS", "TaskState", "TaskClient", "build_multipart", "TRACE_MAX_CHARS", "TaskState", "TaskClient", "build_multipart",
"HttpTaskClient", "CompletedTask", "run_task", "HttpTaskClient", "CompletedTask", "run_task",
"CodexProbeAdapter", "CodexEditor", "CodexSelector", "CodexReflector", "CodexProbeAdapter", "CodexEditor", "CodexSelector", "CodexReflector",
"run_improvement_via_tasks", "validate_run_tag", "preflight_checks", "run_improvement_via_tasks", "run_sweep", "validate_run_tag", "preflight_checks",
] ]
+99
View File
@@ -445,6 +445,22 @@ def _thread_worktree_for_task(server, task):
return os.path.join(base, safe) return os.path.join(base, safe)
def _safe_worktree_path(worktree, rel):
"""worktree 内相对路径的安全解析:空串返回 worktree 本身;
绝对路径、.. 逃逸、符号链接逃逸一律返回 None。realpath 后
再校验 commonpath,目录内的符号链接指回内部时放行(读到
的仍是 worktree 内的内容),指出去时拦住。"""
if not rel:
return worktree
if os.path.isabs(rel):
return None
base = os.path.realpath(worktree)
target = os.path.realpath(os.path.join(base, rel))
if os.path.commonpath([base, target]) != base:
return None
return target
def _extract_thread_list(result): def _extract_thread_list(result):
"""从 thread-list 的 CLI 结果里取 thread 列表:标准形态是 """从 thread-list 的 CLI 结果里取 thread 列表:标准形态是
ThreadListResponse 的 data;threads 键与 result 包裹是 ThreadListResponse 的 data;threads 键与 result 包裹是
@@ -2152,6 +2168,8 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
self._list_followups(user_id, rest[: -len("/followups")]) self._list_followups(user_id, rest[: -len("/followups")])
elif rest.endswith("/timeline") and method == "GET": elif rest.endswith("/timeline") and method == "GET":
self._task_timeline(user_id, rest[: -len("/timeline")]) self._task_timeline(user_id, rest[: -len("/timeline")])
elif rest.endswith("/files") and method == "GET":
self._task_files(user_id, rest[: -len("/files")], query)
elif rest.endswith("/chat-history") and method == "GET": elif rest.endswith("/chat-history") and method == "GET":
self._task_chat_history( self._task_chat_history(
user_id, rest[: -len("/chat-history")]) user_id, rest[: -len("/chat-history")])
@@ -4698,6 +4716,87 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
for r in cp_rows], for r in cp_rows],
}) })
def _task_files(self, user_id, task_id, query):
"""GET /tasks/{id}/files:读本地任务 worktree 里的文件。
只做 local 执行器任务(复用 thread 操作的执行器门):
Docker 任务的 worktree 在容器内、且租户 Docker 下 /app/work
是整项目共享目录,按任务归属读文件会错指,一律 409 明
说;cloud 任务与跨机 worktree 同理。executor 为空的老
任务放行(与 thread 门同款),解析不到 worktree 时 404。
?path=<相对路径>:不传列 worktree 顶层(递归、最多 200
条,只含文件路径与大小,不跟符号链接);传目录列该目录
(非递归,含 is_dir);传文件读文本内容(UTF-8、≤256KB,
二进制或超大一律 409/413 明说,不截断冒充全文)。路径
解析一律走 _safe_worktree_path,防 .. 与符号链接逃逸。
读操作不限制任务状态(进行中任务读到的是中间态)。
"""
task, err = self._thread_task(user_id, task_id, "read")
if err is not None:
self._send_json(err[0], {"error": err[1]})
return
guard = self._thread_task_guard(task)
if guard is not None:
self._send_json(guard[0], {"error": guard[1]})
return
worktree = _thread_worktree_for_task(self.server, task)
if not os.path.isdir(worktree):
self._send_json(404, {"error": "no local worktree for this task"})
return
rel = (query.get("path") or [""])[0]
target = _safe_worktree_path(worktree, rel)
if target is None:
self._send_json(400, {"error": "path escapes task worktree"})
return
if os.path.isdir(target):
entries = []
if not rel:
for root, dirs, files in os.walk(target):
dirs[:] = [d for d in dirs
if not os.path.islink(os.path.join(root, d))]
dirs.sort()
for name in sorted(files):
fpath = os.path.join(root, name)
if os.path.islink(fpath):
continue
entries.append({
"path": os.path.relpath(fpath, worktree),
"size": os.path.getsize(fpath),
})
if len(entries) >= 200:
break
if len(entries) >= 200:
break
else:
for name in sorted(os.listdir(target)):
fpath = os.path.join(target, name)
is_link = os.path.islink(fpath)
entries.append({
"path": os.path.relpath(fpath, worktree),
"size": os.path.getsize(fpath)
if os.path.isfile(fpath) and not is_link else 0,
"is_dir": os.path.isdir(fpath) and not is_link,
})
self._send_json(200, {"task_id": task_id, "path": rel,
"entries": entries})
return
if not os.path.isfile(target):
self._send_json(404, {"error": "no such file"})
return
size = os.path.getsize(target)
if size > 256 * 1024:
self._send_json(413, {"error": "file too large for inline read"})
return
try:
with open(target, encoding="utf-8") as fh:
content = fh.read()
except (OSError, UnicodeDecodeError):
self._send_json(409, {"error": "file is not readable UTF-8 text"})
return
self._send_json(200, {"task_id": task_id, "path": rel,
"size": size, "content": content})
def _task_chat_history(self, user_id, task_id): def _task_chat_history(self, user_id, task_id):
"""GET /tasks/{id}/chat-history:真正的交替历史,一次性 """GET /tasks/{id}/chat-history:真正的交替历史,一次性
按时间交错拼好返回,前端不再猜顺序。 按时间交错拼好返回,前端不再猜顺序。
+201 -10
View File
@@ -19,26 +19,26 @@ from pathlib import Path
try: try:
from cloud import harness2_tasks from cloud import harness2_tasks
from cloud.harness2 import ( from cloud.harness2 import (
parse_choice, parse_experience, parse_harness_files, parse_no_change, parse_choice, parse_deliverable_files, parse_experience,
parse_notes, parse_why, parse_harness_files, parse_no_change, parse_notes, parse_why,
) )
from cloud.harness2_tasks import ( from cloud.harness2_tasks import (
CodexEditor, CodexProbeAdapter, HttpTaskClient, TaskState, CodexEditor, CodexProbeAdapter, HttpTaskClient, TaskState,
build_multipart, preflight_checks, run_improvement_via_tasks, run_task, build_multipart, preflight_checks, run_improvement_via_tasks, run_sweep,
unwrap_result_text, validate_run_tag, run_task, unwrap_result_text, validate_run_tag,
) )
from cloud.uploads import parse_multipart, parse_paths_field from cloud.uploads import parse_multipart, parse_paths_field
except ImportError: # discover -s cloud/tests 时把项目根目录补进 sys.path except ImportError: # discover -s cloud/tests 时把项目根目录补进 sys.path
sys.path.insert(0, str(Path(__file__).resolve().parents[2])) sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
from cloud import harness2_tasks from cloud import harness2_tasks
from cloud.harness2 import ( from cloud.harness2 import (
parse_choice, parse_experience, parse_harness_files, parse_no_change, parse_choice, parse_deliverable_files, parse_experience,
parse_notes, parse_why, parse_harness_files, parse_no_change, parse_notes, parse_why,
) )
from cloud.harness2_tasks import ( from cloud.harness2_tasks import (
CodexEditor, CodexProbeAdapter, HttpTaskClient, TaskState, CodexEditor, CodexProbeAdapter, HttpTaskClient, TaskState,
build_multipart, preflight_checks, run_improvement_via_tasks, run_task, build_multipart, preflight_checks, run_improvement_via_tasks, run_sweep,
unwrap_result_text, validate_run_tag, run_task, unwrap_result_text, validate_run_tag,
) )
from cloud.uploads import parse_multipart, parse_paths_field from cloud.uploads import parse_multipart, parse_paths_field
@@ -61,15 +61,22 @@ LEDGER = b"amount\n100.25\n-20.10\n5.00\n"
class FakeTaskClient: class FakeTaskClient:
"""进程内控制面:按 prompt 类型分派到确定性假 worker。""" """进程内控制面:按 prompt 类型分派到确定性假 worker。"""
def __init__(self, editor_mode="fix", selector_mode="signed", envelope=False): def __init__(self, editor_mode="fix", selector_mode="signed", envelope=False,
fail_tags=(), endpoint_files=None, endpoint_error=False,
probe_extra=""):
self.editor_mode = editor_mode self.editor_mode = editor_mode
self.selector_mode = selector_mode self.selector_mode = selector_mode
self.envelope = envelope self.envelope = envelope
self.fail_tags = tuple(fail_tags)
self.endpoint_files = dict(endpoint_files or {})
self.endpoint_error = endpoint_error
self.probe_extra = probe_extra
self.uploads = {} self.uploads = {}
self.tasks = {} self.tasks = {}
self.by_key = {} self.by_key = {}
self.created = 0 self.created = 0
self.cancelled = [] self.cancelled = []
self.task_models = {}
def upload(self, files): def upload(self, files):
uid = f"u{len(self.uploads) + 1}" uid = f"u{len(self.uploads) + 1}"
@@ -78,6 +85,8 @@ class FakeTaskClient:
def create_task(self, *, project_id, prompt, upload_id=None, def create_task(self, *, project_id, prompt, upload_id=None,
idempotency_key=None, **kwargs): idempotency_key=None, **kwargs):
if idempotency_key and any(t in idempotency_key for t in self.fail_tags):
raise RuntimeError("injected task failure")
if idempotency_key and idempotency_key in self.by_key: if idempotency_key and idempotency_key in self.by_key:
return self.by_key[idempotency_key] return self.by_key[idempotency_key]
tid = f"t{len(self.tasks) + 1}" tid = f"t{len(self.tasks) + 1}"
@@ -94,6 +103,7 @@ class FakeTaskClient:
}) })
self.tasks[tid] = TaskState(task_id=tid, status="completed", self.tasks[tid] = TaskState(task_id=tid, status="completed",
result_text=result) result_text=result)
self.task_models[tid] = kwargs.get("model")
if idempotency_key: if idempotency_key:
self.by_key[idempotency_key] = tid self.by_key[idempotency_key] = tid
return tid return tid
@@ -163,11 +173,22 @@ class FakeTaskClient:
[sys.executable, str(root / "scripts" / "summarize.py"), [sys.executable, str(root / "scripts" / "summarize.py"),
str(root / "ledger.csv")], str(root / "ledger.csv")],
capture_output=True, text=True, timeout=10) capture_output=True, text=True, timeout=10)
return proc.stdout.strip() return proc.stdout.strip() + self.probe_extra
def get_task(self, task_id): def get_task(self, task_id):
return self.tasks[task_id] return self.tasks[task_id]
def get_task_files(self, task_id, path=""):
if self.endpoint_error:
raise RuntimeError("409 files unavailable")
if not path:
return {"entries": [
{"path": p, "size": len(c.encode("utf-8"))}
for p, c in sorted(self.endpoint_files.items())]}
if path in self.endpoint_files:
return {"content": self.endpoint_files[path]}
raise RuntimeError("no such file")
def get_trace_text(self, task_id): def get_trace_text(self, task_id):
return "[ACTION] fake probe\n[OBSERVATION]\n" + self.tasks[task_id].result_text return "[ACTION] fake probe\n[OBSERVATION]\n" + self.tasks[task_id].result_text
@@ -192,6 +213,22 @@ class TestParseHelpers(unittest.TestCase):
self.assertIsNone(parse_choice("no choice here")) self.assertIsNone(parse_choice("no choice here"))
self.assertEqual(parse_why("<why>because</why>"), "because") self.assertEqual(parse_why("<why>because</why>"), "because")
self.assertEqual(parse_notes("<notes>did x</notes>"), "did x") self.assertEqual(parse_notes("<notes>did x</notes>"), "did x")
def test_parse_deliverable_files_lenient(self):
text = ("=== DELIVERABLE FILE: a.md ===\nhello\n"
"=== END DELIVERABLE FILE ===\n"
"=== DELIVERABLE FILE: ../evil ===\nx\n"
"=== END DELIVERABLE FILE ===\n"
"=== DELIVERABLE FILE: b.md ===\nunterminated\n")
self.assertEqual(parse_deliverable_files(text), {"a.md": "hello\n"})
many = "".join(
f"=== DELIVERABLE FILE: f{i}.md ===\nx\n"
"=== END DELIVERABLE FILE ===\n" for i in range(10))
self.assertEqual(len(parse_deliverable_files(many)), 8)
big = ("=== DELIVERABLE FILE: big.md ===\n" + "x" * 70000 + "\n"
"=== END DELIVERABLE FILE ===\n")
content = parse_deliverable_files(big)["big.md"]
self.assertTrue(content.endswith("...[truncated]"))
exp = parse_experience( exp = parse_experience(
"<experience>\ncue: c\nlesson: l\nevidence: e\nconfidence: high\n</experience>") "<experience>\ncue: c\nlesson: l\nevidence: e\nconfidence: high\n</experience>")
self.assertEqual((exp["cue"], exp["lesson"], exp["evidence"]), ("c", "l", "e")) self.assertEqual((exp["cue"], exp["lesson"], exp["evidence"]), ("c", "l", "e"))
@@ -238,6 +275,109 @@ class TestFullRunViaTasks(unittest.TestCase):
self.assertIsNotNone(result.library_entry) self.assertIsNotNone(result.library_entry)
self.assertTrue((root / "library" / "accounting" / "INDEX.md").is_file()) self.assertTrue((root / "library" / "accounting" / "INDEX.md").is_file())
def _models_by_role(self, client):
roles = {}
for key, tid in client.by_key.items():
for role in ("probe", "edit", "select", "reflect"):
if f"-{role}-" in key or key.endswith(f"-{role}"):
roles.setdefault(role, set()).add(client.task_models[tid])
return roles
def test_role_models_land_on_each_role(self):
client = FakeTaskClient()
tmp = tempfile.TemporaryDirectory()
self.addCleanup(tmp.cleanup)
root = Path(tmp.name)
result = run_improvement_via_tasks(
client, project_id="p1", task_id="ledger-1",
prompt="Sum every signed amount in ledger.csv. Return Total: with two decimals.",
base_harness=dict(BASE_TREE), run_tag="roles1",
work_dir=root / "work", domain="accounting",
source_files={"ledger.csv": LEDGER},
k=2, mode="parallel", library_root=root / "library",
model="shared-model",
role_models={"probe": "probe-model", "editor": "editor-model",
"selector": "selector-model",
"reflector": "reflector-model"})
self.assertTrue(result.ok)
roles = self._models_by_role(client)
self.assertEqual(roles["probe"], {"probe-model"})
self.assertEqual(roles["edit"], {"editor-model"})
self.assertEqual(roles["select"], {"selector-model"})
self.assertEqual(roles["reflect"], {"reflector-model"})
def test_role_model_falls_back_to_shared_then_default(self):
client = FakeTaskClient()
tmp = tempfile.TemporaryDirectory()
self.addCleanup(tmp.cleanup)
root = Path(tmp.name)
result = run_improvement_via_tasks(
client, project_id="p1", task_id="ledger-1",
prompt="Sum every signed amount in ledger.csv. Return Total: with two decimals.",
base_harness=dict(BASE_TREE), run_tag="roles2",
work_dir=root / "work", domain="accounting",
source_files={"ledger.csv": LEDGER},
k=1, mode="parallel", library_root=root / "library",
model="shared-model",
role_models={"editor": "editor-model"})
self.assertTrue(result.ok)
roles = self._models_by_role(client)
self.assertEqual(roles["probe"], {"shared-model"})
self.assertEqual(roles["edit"], {"editor-model"})
self.assertEqual(roles["reflect"], {"shared-model"})
# 两者都空时不传模型(服务端默认),假客户端记为 None。
client2 = FakeTaskClient()
result2, _, _ = self._run(client2, run_tag="roles3")
self.assertTrue(result2.ok)
self.assertEqual(set(client2.task_models.values()), {None})
def test_probe_deliverable_files_from_endpoint(self):
client = FakeTaskClient(endpoint_files={
"report.md": "# R\n",
"ledger.csv": "amount\n999.00\n",
"AGENTS.md": BASE_TREE["AGENTS.md"],
})
result, _, root = self._run(client, run_tag="assets1")
self.assertTrue(result.ok)
agg = result.mode_result.rollouts["agg1"]
# 新增文件与改动过的 seed 文件进产物;内容与 seed 一致的不进。
self.assertEqual(agg.deliverable_files,
{"report.md": "# R\n",
"ledger.csv": "amount\n999.00\n"})
found = [p for p in root.rglob("report.md")
if "deliverables/agg1/files" in str(p)]
self.assertTrue(found)
self.assertEqual(found[0].read_text(encoding="utf-8"), "# R\n")
def test_probe_deliverable_files_from_reply_blocks(self):
block = ("\n=== DELIVERABLE FILE: page.html ===\n<h1>hi</h1>\n"
"=== END DELIVERABLE FILE ===\n")
client = FakeTaskClient(endpoint_error=True, probe_extra=block)
result, _, _ = self._run(client, run_tag="assets2")
self.assertTrue(result.ok)
self.assertEqual(
result.mode_result.rollouts["agg1"].deliverable_files,
{"page.html": "<h1>hi</h1>\n"})
def test_probe_deliverable_files_endpoint_wins_on_collision(self):
block = ("\n=== DELIVERABLE FILE: page.html ===\nfrom-reply\n"
"=== END DELIVERABLE FILE ===\n")
client = FakeTaskClient(
endpoint_files={"page.html": "from-endpoint\n"},
probe_extra=block)
result, _, _ = self._run(client, run_tag="assets3")
self.assertTrue(result.ok)
self.assertEqual(
result.mode_result.rollouts["agg1"].deliverable_files,
{"page.html": "from-endpoint\n"})
def test_probe_deliverable_files_endpoint_error_is_not_fatal(self):
client = FakeTaskClient(endpoint_error=True)
result, _, _ = self._run(client, run_tag="assets4")
self.assertTrue(result.ok)
self.assertEqual(
result.mode_result.rollouts["agg1"].deliverable_files, {})
def test_same_run_tag_replays_without_new_tasks(self): def test_same_run_tag_replays_without_new_tasks(self):
client = FakeTaskClient() client = FakeTaskClient()
result1, _, _ = self._run(client, work_sub="work1") result1, _, _ = self._run(client, work_sub="work1")
@@ -563,5 +703,56 @@ class TestHttpClientRetry(unittest.TestCase):
self.assertEqual(counts["n"], 1) self.assertEqual(counts["n"], 1)
class TestSweep(unittest.TestCase):
def _spec(self, tag, **over):
spec = {
"run_tag": tag, "task_id": "ledger-1",
"prompt": "Sum every signed amount in ledger.csv. Return Total: with two decimals.",
"base_harness": dict(BASE_TREE),
"source_files": {"ledger.csv": LEDGER},
"k": 1, "mode": "parallel", "domain": "accounting",
}
spec.update(over)
return spec
def test_sweep_isolates_failures_and_shares_library(self):
client = FakeTaskClient(fail_tags=("bad1",))
tmp = tempfile.TemporaryDirectory()
self.addCleanup(tmp.cleanup)
root = Path(tmp.name)
report = run_sweep(
client, project_id="p1",
runs=[self._spec("good1"), self._spec("bad1"),
self._spec("badk", k=99), self._spec("good2")],
work_root=root / "sweep", library_root=root / "library")
by_tag = {e["run_tag"]: e for e in report["runs"]}
self.assertTrue(by_tag["good1"]["ok"])
self.assertEqual(by_tag["good1"]["chosen_label"], "agg1")
# bad1:全部任务注入失败 → 轮次 ok=False,但不中断后续轮。
self.assertFalse(by_tag["bad1"]["ok"])
# badk:k 越界在方法层抛错 → 记入 error,同样不中断。
self.assertFalse(by_tag["badk"]["ok"])
self.assertIn("ValueError", by_tag["badk"]["error"])
self.assertTrue(by_tag["good2"]["ok"])
self.assertEqual(report["totals"]["n_runs"], 4)
self.assertEqual(report["totals"]["n_ok"], 2)
# 共享库:两轮好结果都入库,同一 INDEX 可见。
self.assertIsNotNone(by_tag["good1"]["library_entry"])
self.assertIsNotNone(by_tag["good2"]["library_entry"])
self.assertTrue((root / "library" / "accounting" / "INDEX.md").is_file())
def test_sweep_requires_run_tag(self):
client = FakeTaskClient()
tmp = tempfile.TemporaryDirectory()
self.addCleanup(tmp.cleanup)
root = Path(tmp.name)
spec = self._spec("good1")
del spec["run_tag"]
report = run_sweep(client, project_id="p1", runs=[spec],
work_root=root / "sweep")
self.assertFalse(report["runs"][0]["ok"])
self.assertIn("run_tag", report["runs"][0]["error"])
if __name__ == "__main__": if __name__ == "__main__":
unittest.main() unittest.main()
+165
View File
@@ -0,0 +1,165 @@
"""GET /tasks/{id}/files 端点测试:本地 worktree 文件读取。
只用标准库 http.client,不跑真实模型,不联网。
任务行直接插库(不走 POST /tasks,避免 worker 派发),
worktree 用 server.host_worktrees_dir 注入到临时目录。
"""
import http.client
import json
import os
import sqlite3
import sys
import tempfile
import threading
import unittest
from pathlib import Path
from urllib.parse import quote
try:
from cloud.auth import AuthStore
from cloud.server import create_server
except ImportError:
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
from cloud.auth import AuthStore
from cloud.server import create_server
class TaskFilesTestCase(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.db_path = os.path.join(self.tmp.name, "control.db")
self.auth = AuthStore(self.db_path)
self.owner_id, self.owner_key = self.auth.create_user("owner")
self.viewer_id, self.viewer_key = self.auth.create_user("viewer")
self.outsider_id, self.outsider_key = self.auth.create_user("outsider")
self.server = create_server(self.db_path, "127.0.0.1", 0)
self.worktrees = os.path.join(self.tmp.name, "host-worktrees")
os.makedirs(self.worktrees)
self.server.host_worktrees_dir = self.worktrees
self.port = self.server.server_address[1]
self.thread = threading.Thread(
target=self.server.serve_forever, daemon=True)
self.thread.start()
status, self.project = self._req(
"POST", "/projects", {"name": "demo"}, self.owner_key)
self.assertEqual(status, 201)
self.auth.set_member(
self.project["id"], self.owner_id, self.viewer_id, "viewer")
self._task("task-files-1", executor=None)
wt = os.path.join(self.worktrees, "task-files-1")
os.makedirs(os.path.join(wt, "sub"))
Path(wt, "report.md").write_text("# Report\n", encoding="utf-8")
Path(wt, "sub", "page.html").write_text("<h1>hi</h1>\n",
encoding="utf-8")
# 非法 UTF-8 字节(\x00 是合法 UTF-8,不能用来测二进制拒绝)。
Path(wt, "blob.bin").write_bytes(b"\x89PNG\r\n\x1a\n\xff\xfe")
Path(wt, "big.txt").write_text("x" * (256 * 1024 + 1),
encoding="utf-8")
os.symlink(os.path.join(self.tmp.name, "outside.txt"),
os.path.join(wt, "escape"))
Path(self.tmp.name, "outside.txt").write_text("secret\n",
encoding="utf-8")
self._task("task-docker-1", executor="docker")
self._task("task-nowt-1", executor=None)
def tearDown(self):
self.server.shutdown()
self.server.server_close()
self.thread.join(timeout=5)
try:
self.auth.conn.close()
except Exception:
pass
self.tmp.cleanup()
def _task(self, task_id, executor):
conn = sqlite3.connect(self.db_path)
try:
conn.execute(
"INSERT INTO tasks(id, project_id, prompt, status, executor)"
" VALUES(?, ?, ?, ?, ?)",
(task_id, self.project["id"], "do it", "completed", executor))
conn.commit()
finally:
conn.close()
def _req(self, method, path, body=None, key=None):
conn = http.client.HTTPConnection("127.0.0.1", self.port, timeout=10)
hdrs = {}
payload = None
if body is not None:
payload = json.dumps(body).encode("utf-8")
hdrs["Content-Type"] = "application/json"
if key is not None:
hdrs["X-API-Key"] = key
try:
conn.request(method, path, body=payload, headers=hdrs)
resp = conn.getresponse()
data = resp.read()
finally:
conn.close()
try:
parsed = json.loads(data.decode("utf-8")) if data else None
except ValueError:
parsed = None
return resp.status, parsed
def _files(self, task_id, path=None, key=None):
url = f"/tasks/{task_id}/files"
if path is not None:
url += "?path=" + quote(path)
return self._req("GET", url, key=self.owner_key if key is None else key)
def test_list_top_level_recursive(self):
status, body = self._files("task-files-1")
self.assertEqual(status, 200, body)
paths = sorted(e["path"] for e in body["entries"])
self.assertIn("report.md", paths)
self.assertIn("sub/page.html", paths)
self.assertNotIn("escape", paths) # 符号链接不跟进列表
def test_read_file_and_subdir_listing(self):
status, body = self._files("task-files-1", "report.md")
self.assertEqual(status, 200, body)
self.assertEqual(body["content"], "# Report\n")
status, body = self._files("task-files-1", "sub")
self.assertEqual(status, 200, body)
self.assertEqual([e["path"] for e in body["entries"]],
["sub/page.html"])
def test_path_escape_blocked(self):
for bad in ("../outside.txt", "/etc/passwd", "escape", "sub/../../x"):
status, body = self._files("task-files-1", bad)
self.assertIn(status, (400, 404), (bad, body))
# escape 是指到 worktree 外的符号链接:必须 400,不是读到 secret。
status, body = self._files("task-files-1", "escape")
self.assertEqual(status, 400, body)
def test_missing_file_and_missing_worktree(self):
status, _ = self._files("task-files-1", "nope.md")
self.assertEqual(status, 404)
status, _ = self._files("task-nowt-1")
self.assertEqual(status, 404)
status, _ = self._files("task-unknown")
self.assertEqual(status, 404)
def test_binary_and_oversized(self):
status, body = self._files("task-files-1", "blob.bin")
self.assertEqual(status, 409, body)
status, body = self._files("task-files-1", "big.txt")
self.assertEqual(status, 413, body)
def test_docker_task_409_and_auth(self):
status, body = self._files("task-docker-1")
self.assertEqual(status, 409, body)
status, _ = self._req("GET", "/tasks/task-files-1/files")
self.assertEqual(status, 401)
status, _ = self._files("task-files-1", key=self.outsider_key)
self.assertEqual(status, 403)
status, _ = self._files("task-files-1", key=self.viewer_key)
self.assertEqual(status, 200)
if __name__ == "__main__":
unittest.main()
+3 -3
View File
@@ -34,9 +34,9 @@ benchmark 上的,不是本项目的承诺——本项目没有 benchmark,接
| 经验库(分领域、整树 + EXPERIENCE.md + INDEX.md) | 照搬存储格式,改写生效语义:入库 ≠ 自动生效(见下) | | 经验库(分领域、整树 + EXPERIENCE.md + INDEX.md) | 照搬存储格式,改写生效语义:入库 ≠ 自动生效(见下) |
| Ledger(会话/探针账本)与成本 | 改写:成本汇总进 `ModeResult.total_cost_usd`,由调用方接现有 `cloud/cost.py` / 租户配额计费;另有 `budget_usd` 超支门与 `max_sessions` 会话数门(计探针 + 阶段尝试会话;edit 内 fmt2 纠正与选择/反思会话不计入):任一超支后不再调用编辑器/探针,剩余阶段回落,且该轮不入库;上游的 R=4k/P=3k 事后强校验未引(本项目结构与上游计数不完全同构,门控代替断言) | | Ledger(会话/探针账本)与成本 | 改写:成本汇总进 `ModeResult.total_cost_usd`,由调用方接现有 `cloud/cost.py` / 租户配额计费;另有 `budget_usd` 超支门与 `max_sessions` 会话数门(计探针 + 阶段尝试会话;edit 内 fmt2 纠正与选择/反思会话不计入):任一超支后不再调用编辑器/探针,剩余阶段回落,且该轮不入库;上游的 R=4k/P=3k 事后强校验未引(本项目结构与上游计数不完全同构,门控代替断言) |
| run_tag 形状与建任务前预检 | 对照补齐:`validate_run_tag`(字符集、首字符不为 -、不以 s 数字段开头)与 `preflight_checks`(base 目录与 AGENTS.md、source 目录、提示、tag、控制面 health 探针);CLI `--check` 只预检不建任务 | | run_tag 形状与建任务前预检 | 对照补齐:`validate_run_tag`(字符集、首字符不为 -、不以 s 数字段开头)与 `preflight_checks`(base 目录与 AGENTS.md、source 目录、提示、tag、控制面 health 探针);CLI `--check` 只预检不建任务 |
| 分角色模型(SOLVER / IMPROVER / JUDGE) | 未引:全部角色共用调用方透传的同一个 model;需用户定各角色模型才有意义 | | 分角色模型(SOLVER / IMPROVER / JUDGE) | 已引(机制):`run_improvement_via_tasks` 的 `role_models`(probe / editor / selector / reflector)+ CLI `--model-<角色>`;非空值覆盖该角色,缺键回落共享 `--model`,两者都空走服务端默认——默认行为不变,模型选择权仍在调用方(有价模型产生真实成本) |
| 产物资产(deliverable_files / deliverable_assets / images)与选择 assets note | 未引:探针交付物目前只有最终回复文本;需新增 worktree 文件读取端点后与该端点一起做 | | 产物资产(deliverable_files / deliverable_assets / images)与选择 assets note | 已引(两层):①服务端 `GET /tasks/{id}/files`(本地任务 worktree 读取,路径安全解析、文本 ≤256KB、列表 ≤200 条;Docker/跨机任务 409);②探针回复里的 `DELIVERABLE FILE` 块(所有执行器可用,端点结果优先、坏块丢弃不炸轮次)。产物进 `Rollout.deliverable_files`,并写入选择/编辑/反思上下文的 `deliverables/<label>/files/`。边界:租户 Docker 下 `/app/work` 是整项目共享目录,按任务归属读文件会错指,故端点对 Docker 任务只 409、不给错数据;此时只有 agent 自报层可用。images(二进制产物)未引:端点只读 UTF-8 文本 |
| sweep 层(多任务批量、跨任务 library restore、dead 隔离、report 汇总) | 未引:单任务产品形态用不上,批量改进需求出现再做 | | sweep 层(多任务批量、跨任务 library restore、dead 隔离、report 汇总) | 已引:`run_sweep` + CLI `examples/harness2_sweep.py`(共享 library_root、单轮失败隔离、sweep-report.json 汇总;manifest 预检不过不建任务,默认只打印计划,`--yes` 才真跑) |
| 提示词 | 改写短版(中文项目语境下的英文任务提示,结构同上游,措辞自写),不抄上游长提示 | | 提示词 | 改写短版(中文项目语境下的英文任务提示,结构同上游,措辞自写),不抄上游长提示 |
| Benchmark 适配器(JobBench / LAB / WorkBuddy)、评分器、rubric | 不引。任务就是一个普通 codex 任务,delivered 由 adapter 定义(如产物存在且非空) | | Benchmark 适配器(JobBench / LAB / WorkBuddy)、评分器、rubric | 不引。任务就是一个普通 codex 任务,delivered 由 adapter 定义(如产物存在且非空) |
| Proposer 通道(Claude CLI / OpenCode)、Vertex 配置 | 不引。编辑器/选择器/反思器是调用方注入的协议,真实接线时是 codex 任务(见下) | | Proposer 通道(Claude CLI / OpenCode)、Vertex 配置 | 不引。编辑器/选择器/反思器是调用方注入的协议,真实接线时是 codex 任务(见下) |
+9 -1
View File
@@ -69,6 +69,10 @@ def main():
ap.add_argument("--output", default="") ap.add_argument("--output", default="")
ap.add_argument("--library-root", default="") ap.add_argument("--library-root", default="")
ap.add_argument("--model", default=None) ap.add_argument("--model", default=None)
ap.add_argument("--model-probe", default=None)
ap.add_argument("--model-editor", default=None)
ap.add_argument("--model-selector", default=None)
ap.add_argument("--model-reflector", default=None)
ap.add_argument("--effort", default=None) ap.add_argument("--effort", default=None)
ap.add_argument("--approval", default=None) ap.add_argument("--approval", default=None)
ap.add_argument("--environment-id", default=None) ap.add_argument("--environment-id", default=None)
@@ -148,7 +152,11 @@ def main():
k=args.k, mode=args.mode, k=args.k, mode=args.mode,
library_root=Path(args.library_root) if args.library_root else None, library_root=Path(args.library_root) if args.library_root else None,
budget_usd=args.budget, max_sessions=args.max_sessions, budget_usd=args.budget, max_sessions=args.max_sessions,
model=args.model, effort=args.effort, model=args.model,
role_models={"probe": args.model_probe, "editor": args.model_editor,
"selector": args.model_selector,
"reflector": args.model_reflector},
effort=args.effort,
approval=args.approval, environment_id=args.environment_id, approval=args.approval, environment_id=args.environment_id,
executor=args.executor, no_time_limit=args.no_time_limit, executor=args.executor, no_time_limit=args.no_time_limit,
timeout_seconds=args.timeout) timeout_seconds=args.timeout)
+176
View File
@@ -0,0 +1,176 @@
"""多轮 Harness² 改进批量跑(sweep,接线层 CLI)。
读一个 JSON manifest,顺序跑多轮 run_improvement_via_tasks:所有轮
共享同一个 library_root(已入库的经验会被后续轮读到,即上游 sweep
的 library restore 语义);单轮失败只记入报告、不中断后续轮
(dead 隔离);最后写 sweep-report.json 并汇总打印。
这会创建真实任务、花真实模型配额——所以默认只打印计划,不建
任务:先跑一遍不带 --yes 看计划,确认无误后,再带 --yes 重跑。
密钥不走命令行参数(ps 可见):用环境变量 CODEX_API_KEY。
manifest 形状:
{
"runs": [
{"run_tag": "sw1", "task_id": "ledger-1",
"prompt": "Sum every signed amount in ledger.csv.",
"base_harness_dir": "./base-harness", "source_dir": "./task-source",
"domain": "accounting", "k": 1, "mode": "parallel",
"budget": 5, "max_sessions": null, "model": null,
"role_models": null, "effort": null, "approval": null,
"environment_id": null, "executor": null,
"no_time_limit": false, "timeout": 3600}
]
}
prompt 也可用 prompt_file(相对 manifest 所在目录);base_harness_dir
与 source_dir 同样相对 manifest 目录解析(绝对路径原样用)。
"""
import argparse
import json
import os
import sys
import tempfile
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from cloud.harness2 import read_tree # noqa: E402
from cloud.harness2_tasks import ( # noqa: E402
HttpTaskClient, preflight_checks, run_sweep, validate_run_tag,
)
def _flatten(root: Path) -> dict[str, bytes]:
out: dict[str, bytes] = {}
if not root or not root.is_dir():
return out
for p in sorted(root.rglob("*")):
if p.is_file() and not p.is_symlink() and "__pycache__" not in p.parts:
out[str(p.relative_to(root))] = p.read_bytes()
return out
def _resolve(manifest_dir: Path, value):
if not value:
return value
p = Path(value)
return p if p.is_absolute() else manifest_dir / p
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--manifest", required=True, help="JSON manifest 路径")
ap.add_argument("--base-url", required=True, help="控制面地址")
ap.add_argument("--project-id", required=True)
ap.add_argument("--output", default="",
help="sweep 工作根目录(默认临时目录)")
ap.add_argument("--library-root", default="",
help="共享经验库根目录(空 = 各轮不入库)")
ap.add_argument("--yes", action="store_true", help="确认创建真实任务并消费配额")
args = ap.parse_args()
manifest_path = Path(args.manifest)
try:
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
except (OSError, ValueError) as exc:
print(f"manifest 读不到或不是合法 JSON:{exc}", file=sys.stderr)
return 2
runs = manifest.get("runs")
if not isinstance(runs, list) or not runs:
print("manifest 里 runs 必须是非空列表", file=sys.stderr)
return 2
manifest_dir = manifest_path.resolve().parent
specs = []
seen_tags = set()
for i, raw in enumerate(runs):
if not isinstance(raw, dict):
print(f"runs[{i}] 不是对象", file=sys.stderr)
return 2
tag = raw.get("run_tag", "")
try:
if not tag:
raise ValueError("每轮必须给 run_tag")
validate_run_tag(tag)
except ValueError as exc:
print(f"runs[{i}] run_tag 非法:{exc}", file=sys.stderr)
return 2
if tag in seen_tags:
print(f"runs[{i}] run_tag 重复:{tag}", file=sys.stderr)
return 2
seen_tags.add(tag)
prompt = raw.get("prompt", "")
if raw.get("prompt_file"):
try:
prompt = _resolve(manifest_dir, raw["prompt_file"]).read_text(
encoding="utf-8")
except OSError as exc:
print(f"runs[{i}] prompt_file 读不到:{exc}", file=sys.stderr)
return 2
base_dir = _resolve(manifest_dir, raw.get("base_harness_dir", ""))
errors, warnings = preflight_checks(
base_harness_dir=base_dir,
source_dir=_resolve(manifest_dir, raw.get("source_dir")) or None,
prompt=prompt, run_tag=tag, base_url=args.base_url)
for e in errors:
print(f"FAIL runs[{i}] {tag}: {e}", file=sys.stderr)
for w in warnings:
print(f"WARN runs[{i}] {tag}: {w}")
if errors:
return 2
source_dir = _resolve(manifest_dir, raw.get("source_dir"))
specs.append({
"run_tag": tag,
"task_id": raw.get("task_id", f"h2-task-{tag}"),
"prompt": prompt,
"base_harness": read_tree(base_dir),
"source_files": _flatten(source_dir) if source_dir else None,
"domain": raw.get("domain", "general"),
"k": raw.get("k", 1),
"mode": raw.get("mode", "parallel"),
"budget_usd": raw.get("budget"),
"max_sessions": raw.get("max_sessions"),
"model": raw.get("model"),
"role_models": raw.get("role_models"),
"effort": raw.get("effort"),
"approval": raw.get("approval"),
"environment_id": raw.get("environment_id"),
"executor": raw.get("executor"),
"no_time_limit": bool(raw.get("no_time_limit", False)),
"timeout_seconds": raw.get("timeout", 3600),
})
print(f"计划:项目 {args.project_id},共 {len(specs)} 轮,"
f"tags={sorted(seen_tags)},"
f"共享 library={'是' if args.library_root else '否(不入库)'}")
if not args.yes:
print("未带 --yes:只打印计划,不创建任何任务。确认无误后加 --yes 重跑。")
return 0
api_key = os.environ.get("CODEX_API_KEY")
if not api_key:
print("缺少 CODEX_API_KEY 环境变量(密钥不接受命令行参数)", file=sys.stderr)
return 2
work_root = Path(args.output) if args.output else Path(
tempfile.mkdtemp(prefix="h2sweep-"))
client = HttpTaskClient(args.base_url, api_key)
report = run_sweep(
client, project_id=args.project_id, runs=specs, work_root=work_root,
library_root=Path(args.library_root) if args.library_root else None)
report_path = work_root / "sweep-report.json"
report_path.write_text(json.dumps(report, ensure_ascii=False, indent=2),
encoding="utf-8")
for entry in report["runs"]:
status = "ok" if entry["ok"] else f"FAIL {entry['error']}"
print(f"{entry['run_tag']}: {status} "
f"chosen={entry['chosen_label']} "
f"cost={entry['total_cost_usd']}")
totals = report["totals"]
print(f"汇总:{totals['n_ok']}/{totals['n_runs']} 轮成功,"
f"总成本 {totals['total_cost_usd']},报告 {report_path}")
return 0 if totals["n_ok"] == totals["n_runs"] else 1
if __name__ == "__main__":
raise SystemExit(main())