harness2: upstream gap fill — trivial selection fix, session budget, run_tag validation, preflight check
- select_final trivial condition now k==1 and single-candidate pool, regardless of whether a selector is provided (previously gated on selector is None, so wired runs always paid for a useless selector task at k=1) - Budget gains max_sessions gate (probe + stage-attempt sessions; fmt2 corrections and selector/reflector sessions exempt), the backstop for unpriced models where the USD gate never binds - validate_run_tag (charset, no leading '-', no s<N> seed prefix) enforced at run_improvement_via_tasks and the CLI - CLI --check preflight (base harness, source dir, prompt, tag, control-plane /health probe) without creating tasks - ModeResult.to_dict now includes rollout summaries so a lost probe is visible in full_result instead of only on the server - CLI plan counts the selector only when k>1
This commit is contained in:
1 parent
3d60f32bd3
commit
4ed1479d7a
5 files changed
+247
-26
No files matched your search
+46
-6
@@ -26,8 +26,9 @@ Claude CLI 通道都没有搬。
|
||||
|
||||
选终版时只比“产物”:候选产物先匿名化(A/B/C……,按任务+k+模式
|
||||
定种子洗牌,可复现),选择器看不到哪个产物出自哪个 harness;选不
|
||||
出来时回落到已交付的终版,绝不抛异常把整轮炸掉。k=1 且没有选择
|
||||
器时直接走 trivial:唯一的终版就是答案,不必再选一次。
|
||||
出来时回落到已交付的终版,绝不抛异常把整轮炸掉。k=1 且终版池
|
||||
只有唯一候选时直接走 trivial:唯一的终版就是答案,不必再选
|
||||
一次(与选择器是否提供无关)。
|
||||
|
||||
经验库(上游叫 harnessing-level recursion):
|
||||
采纳后,反思器只写一条有证据的经验(cue / lesson / evidence 三项
|
||||
@@ -740,16 +741,32 @@ def _mission_experience() -> str:
|
||||
|
||||
@dataclass
|
||||
class Budget:
|
||||
"""美元门 + 会话数门(2026-10-29 对照补齐)。
|
||||
|
||||
上游用会话数记账并强校验期望计数(R=4k、P=3k);本项目此前
|
||||
只有美元门,在 unpriced 模型上成本恒为 0、门永远不触发。
|
||||
sessions 计探针会话与阶段尝试会话;edit() 内部的 fmt2 纠正
|
||||
与选择/反思会话不计入(前者在接线层、后者不走 budget),
|
||||
上限语义因此是“受门控的会话数”,不是全轮任务总数。
|
||||
"""
|
||||
|
||||
max_cost_usd: float | None = None
|
||||
spent_usd: float = 0.0
|
||||
max_sessions: int | None = None
|
||||
sessions: int = 0
|
||||
|
||||
@property
|
||||
def exceeded(self) -> bool:
|
||||
return self.max_cost_usd is not None and self.spent_usd > self.max_cost_usd
|
||||
if self.max_cost_usd is not None and self.spent_usd > self.max_cost_usd:
|
||||
return True
|
||||
return self.max_sessions is not None and self.sessions >= self.max_sessions
|
||||
|
||||
def add(self, cost: float) -> None:
|
||||
self.spent_usd += float(cost or 0.0)
|
||||
|
||||
def add_session(self) -> None:
|
||||
self.sessions += 1
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 阶段:一次提议(含一次修正机会)+ 一次探针
|
||||
@@ -816,6 +833,18 @@ class ModeResult:
|
||||
if any(s.dim == "AGG" for s in self.stages)
|
||||
else None,
|
||||
"errors": self.errors,
|
||||
# 探针结果此前不在 full_result 里:real5 的终版探针没建
|
||||
# 出来时,本地日志完全看不到原因,只能去服务端翻任务。
|
||||
# 只放摘要(不放 trajectory),避免日志爆炸。
|
||||
"rollouts": {
|
||||
lbl: {
|
||||
"ok": r.ok,
|
||||
"delivered": r.delivered,
|
||||
"error": (r.error or "")[:400],
|
||||
"deliverable_text": (r.deliverable_text or "")[:500],
|
||||
}
|
||||
for lbl, r in self.rollouts.items()
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@@ -872,6 +901,11 @@ def _run_stage(
|
||||
err = ""
|
||||
spent = 0.0
|
||||
for attempt in range(1, STAGE_ATTEMPTS + 1):
|
||||
if budget is not None:
|
||||
if budget.exceeded:
|
||||
err = "budget exceeded before this attempt"
|
||||
break
|
||||
budget.add_session()
|
||||
try:
|
||||
result = editor.edit(
|
||||
full_mission + (
|
||||
@@ -918,6 +952,8 @@ def _roll(
|
||||
if budget is not None and budget.exceeded:
|
||||
return Rollout(label=label, ok=False, delivered=False,
|
||||
error="budget exceeded before this rollout")
|
||||
if budget is not None:
|
||||
budget.add_session()
|
||||
try:
|
||||
rv = adapter.execute(label, tree)
|
||||
except Exception as exc:
|
||||
@@ -1164,8 +1200,11 @@ def select_final(
|
||||
f"{text.count(chr(10)) + 1:,} lines. COMPLETE -- nothing trimmed. -->\n")
|
||||
(cand_dir / f"{anon}.md").write_text(header + text, encoding="utf-8")
|
||||
ids = ", ".join(sorted(sel.id_map))
|
||||
if selector is None and k == 1:
|
||||
# 唯一终版:不必再开一轮选择会话(上游的 trivial_selection)。
|
||||
if k == 1 and len(pool) == 1:
|
||||
# 唯一终版:不必再开一轮选择会话(上游的 trivial_selection,
|
||||
# 条件是“终版池 + k=1”,与选择器是否提供无关;此前误写成
|
||||
# selector is None 才 trivial,而接线永远传选择器,导致
|
||||
# k=1 也白白跑一个选择任务——2026-10-29 对照发现)。
|
||||
label = next(iter(pool))
|
||||
sel.ok = True
|
||||
sel.trivial = True
|
||||
@@ -1402,6 +1441,7 @@ def improve_task(
|
||||
library: Library | None = None,
|
||||
auto_apply: bool = False,
|
||||
budget_usd: float | None = None,
|
||||
max_sessions: int | None = None,
|
||||
) -> ImprovementResult:
|
||||
"""跑一轮完整改进。base 探针先跑一次,作为全部候选共享的证据。
|
||||
|
||||
@@ -1413,7 +1453,7 @@ def improve_task(
|
||||
if not 1 <= int(k) <= MAX_K:
|
||||
raise ValueError(f"k must be 1..{MAX_K}, got {k!r}")
|
||||
work_dir = Path(work_dir)
|
||||
budget = Budget(max_cost_usd=budget_usd)
|
||||
budget = Budget(max_cost_usd=budget_usd, max_sessions=max_sessions)
|
||||
base_tree = adapter.base_harness()
|
||||
# 校验 base 本身:base 若已不合法,整轮的对比就没意义,直接报错
|
||||
# 而不是让候选在坏 base 上各跑各的。
|
||||
|
||||
Reference in new issue
Block a user