harness2: recursive harness optimization wired to real codex tasks
Port Google Research's Harness^2 method layer (model weights frozen, only the surrounding harness is optimized): Propose-Probe-Compose with INV/DOM/AGG editors, parallel/sequential modes, per-domain experience library, anonymous-product selection, k<=3 hard cap and budget gate. Execution goes only through the existing control-plane API (upload -> task with idempotency key -> poll -> trace/cost -> cancel on timeout); no new server endpoints, no codex flags composed, tasks inherit the normal executor/sandbox/quota/audit path. Wiring lessons from four real rounds against production (project wwww, ledger task): HttpTaskClient retries transient disconnects and 502/503/429 (3 attempts); editors get a worked file-block example, an explicit <no-change> marker and one feedback retry (.fmt2) instead of silently accepting notes-only replies; run_task unwraps the server's result_text JSON envelope (text field holds the real reply) — without this, line-anchored file-block parsing could never see blocks the agents were actually emitting. Final round (real4) fully green: all stages landed, selection non-fallback, experience入库, probe delivers the correct signed total 85.15. Tests: 35 harness tests (17 method + 18 wiring incl. fake-control-plane full round, idempotent replay, retry, format retry, envelope round). Deliberate deviations from upstream are documented in docs/harness2.md (no network-unrestricted/sandbox-disabled defaults, auto_apply off, no benchmark/scorer, probe deliverable = final reply text).
This commit is contained in:
host committed
1 parent
e48b756f96
commit
42abe3cc5b
7 files changed
+3504
No files matched your search
@@ -0,0 +1,335 @@
|
||||
"""cloud/harness2.py 单测：递归改进循环。
|
||||
|
||||
不跑真实模型、不联网：执行器在进程内跑 harness 树里的脚本，
|
||||
编辑器/选择器/反思器都是确定性假实现（上游 harness2-demo 的同款
|
||||
做法：base 脚本只加正数，编辑器去掉正数过滤，正确总数是带符号和）。
|
||||
|
||||
账本：100.25, -20.10, 5.00 —— 带符号和 85.15，只加正数是 105.25。
|
||||
"""
|
||||
|
||||
import csv
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
try:
|
||||
from cloud import harness2
|
||||
from cloud.harness2 import (
|
||||
Library, Rollout, EditResult, Budget, build_context, validate,
|
||||
read_tree, write_tree, is_allowed, normalize_rel, autotrigger,
|
||||
skill_names, diff_summary, diversity, improve_task, select_final,
|
||||
assign_ids, has_experience,
|
||||
)
|
||||
except ImportError: # discover -s cloud/tests 时把项目根目录补进 sys.path
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||||
from cloud import harness2
|
||||
from cloud.harness2 import (
|
||||
Library, Rollout, EditResult, Budget, build_context, validate,
|
||||
read_tree, write_tree, is_allowed, normalize_rel, autotrigger,
|
||||
skill_names, diff_summary, diversity, improve_task, select_final,
|
||||
assign_ids, has_experience,
|
||||
)
|
||||
|
||||
BASE_SCRIPT = """import csv
|
||||
import sys
|
||||
from decimal import Decimal
|
||||
with open(sys.argv[1]) as f:
|
||||
amounts = [Decimal(r['amount']) for r in csv.DictReader(f)]
|
||||
print(f"Total: {sum((a for a in amounts if a > 0), Decimal(0)):.2f}")
|
||||
"""
|
||||
|
||||
BASE_TREE = {
|
||||
"AGENTS.md": "Use scripts/summarize.py to total the ledger.\n",
|
||||
"scripts/summarize.py": BASE_SCRIPT,
|
||||
}
|
||||
|
||||
|
||||
class LedgerAdapter:
|
||||
"""任务：对账本求和。执行 = 真跑 harness 树里的脚本（无模型）。"""
|
||||
|
||||
task_id = "ledger-1"
|
||||
domain = "accounting"
|
||||
prompt = "Sum every signed amount in ledger.csv. Return Total: with two decimals."
|
||||
|
||||
def __init__(self, root):
|
||||
self.root = Path(root)
|
||||
self.executions = 0
|
||||
self.ledger = self.root / "ledger.csv"
|
||||
self.ledger.write_text("amount\n100.25\n-20.10\n5.00\n", encoding="utf-8")
|
||||
|
||||
def base_harness(self):
|
||||
return dict(BASE_TREE)
|
||||
|
||||
def execute(self, label, harness):
|
||||
self.executions += 1
|
||||
work = self.root / "runs" / label
|
||||
work.mkdir(parents=True, exist_ok=True)
|
||||
write_tree(harness, work / "harness")
|
||||
proc = subprocess.run(
|
||||
[sys.executable, str(work / "harness" / "scripts" / "summarize.py"),
|
||||
str(self.ledger)],
|
||||
capture_output=True, text=True, timeout=10,
|
||||
)
|
||||
out = proc.stdout.strip()
|
||||
return Rollout(
|
||||
label=label, ok=proc.returncode == 0, delivered=out.startswith("Total:"),
|
||||
deliverable_text=out, trajectory="[ACTION] Run scripts/summarize.py\n"
|
||||
"[OBSERVATION]\n" + out, cost_usd=0.01,
|
||||
error="" if proc.returncode == 0 else proc.stderr[-200:],
|
||||
)
|
||||
|
||||
def validate_tree(self, harness):
|
||||
if "scripts/summarize.py" not in harness:
|
||||
raise ValueError("The local executor requires scripts/summarize.py")
|
||||
|
||||
|
||||
class FixEditor:
|
||||
"""确定性编辑器：去掉正数过滤；同时在 context/harness/ 里真实改文件，
|
||||
模拟 codex 任务编辑工作目录的方式，再把改完的树读回。"""
|
||||
|
||||
def edit(self, mission, context_dir, label):
|
||||
harness_dir = Path(context_dir) / "harness"
|
||||
script = harness_dir / "scripts" / "summarize.py"
|
||||
script.write_text(script.read_text().replace(" if a > 0", ""), encoding="utf-8")
|
||||
return EditResult(tree=read_tree(harness_dir),
|
||||
notes="Keep signed amounts: the base total omitted credits.")
|
||||
|
||||
|
||||
class BadEditor:
|
||||
"""永远交回非法树：AGENTS.md 为空，校验必拒，最终必须回落到 base。"""
|
||||
|
||||
def edit(self, mission, context_dir, label):
|
||||
return EditResult(tree={"AGENTS.md": ""}, notes="bad proposal")
|
||||
|
||||
|
||||
class PickSignedSelector:
|
||||
def select(self, mission, context_dir):
|
||||
for path in sorted((Path(context_dir) / "candidates").glob("*.md")):
|
||||
if "Total: 85.15" in path.read_text():
|
||||
return path.stem, "The signed total matches the source ledger.", 0.0
|
||||
raise ValueError("no candidate with the signed total")
|
||||
|
||||
|
||||
class PickFirstSelector:
|
||||
def select(self, mission, context_dir):
|
||||
paths = sorted((Path(context_dir) / "candidates").glob("*.md"))
|
||||
return paths[0].stem, "first delivered candidate", 0.0
|
||||
|
||||
|
||||
class RaisingSelector:
|
||||
def select(self, mission, context_dir):
|
||||
raise RuntimeError("selector session failed")
|
||||
|
||||
|
||||
class LedgerReflector:
|
||||
def reflect(self, mission, context_dir):
|
||||
root = Path(context_dir)
|
||||
changed = (root / "harness" / "scripts" / "summarize.py").read_text() != (
|
||||
root / "base_harness" / "scripts" / "summarize.py").read_text()
|
||||
if not changed:
|
||||
return {}
|
||||
return {
|
||||
"cue": "When a ledger includes credits and debits",
|
||||
"lesson": "Include both signs when summing in scripts/summarize.py.",
|
||||
"evidence": "Removing the positive-only filter changed the probe total "
|
||||
"to include the credits omitted by the base trajectory.",
|
||||
"confidence": "high for this observed arithmetic contrast",
|
||||
"edits": "scripts/summarize.py: dropped the positive-only filter; observed: "
|
||||
"probe total included the omitted credits; verdict: helped",
|
||||
}
|
||||
|
||||
|
||||
class TestTree(unittest.TestCase):
|
||||
def test_validate_accepts_base(self):
|
||||
census = validate(dict(BASE_TREE))
|
||||
self.assertEqual(census["scripts"], 1)
|
||||
self.assertEqual(census["n_files"], 2)
|
||||
|
||||
def test_validate_rejects_skill_without_description(self):
|
||||
tree = dict(BASE_TREE)
|
||||
tree[".agents/skills/check/SKILL.md"] = "Do the check.\n"
|
||||
tree["AGENTS.md"] += "- When checking, read .agents/skills/check/SKILL.md.\n"
|
||||
with self.assertRaises(ValueError) as cm:
|
||||
validate(tree)
|
||||
self.assertIn("description", str(cm.exception))
|
||||
|
||||
def test_validate_rejects_untriggered_skill(self):
|
||||
tree = dict(BASE_TREE)
|
||||
tree[".agents/skills/check/SKILL.md"] = "---\ndescription: check totals\n---\nDo it.\n"
|
||||
with self.assertRaises(ValueError) as cm:
|
||||
validate(tree)
|
||||
self.assertIn("never named", str(cm.exception))
|
||||
|
||||
def test_validate_rejects_bad_hooks_and_bad_script(self):
|
||||
tree = dict(BASE_TREE)
|
||||
tree[".codex/hooks.json"] = '{"hooks": {"Nope": []}}'
|
||||
with self.assertRaises(ValueError):
|
||||
validate(tree)
|
||||
tree = dict(BASE_TREE)
|
||||
tree["scripts/summarize.py"] = "def broken(:\n"
|
||||
with self.assertRaises(ValueError):
|
||||
validate(tree)
|
||||
tree = dict(BASE_TREE)
|
||||
tree["AGENTS.md"] = "x" * 8001
|
||||
with self.assertRaises(ValueError):
|
||||
validate(tree)
|
||||
|
||||
def test_validate_rejects_missing_script_ref(self):
|
||||
tree = dict(BASE_TREE)
|
||||
tree["AGENTS.md"] += "Run python3 scripts/nope.py first.\n"
|
||||
with self.assertRaises(ValueError):
|
||||
validate(tree)
|
||||
|
||||
def test_autotrigger_repairs_and_respects_cap(self):
|
||||
tree = dict(BASE_TREE)
|
||||
tree[".agents/skills/check/SKILL.md"] = "---\ndescription: check totals\n---\nDo it.\n"
|
||||
fixed = autotrigger(tree)
|
||||
self.assertEqual(fixed, ["check"])
|
||||
self.assertIn("check", tree["AGENTS.md"])
|
||||
validate(tree) # 补完触发行后应能通过
|
||||
capped = dict(BASE_TREE)
|
||||
capped["AGENTS.md"] = "x" * 7995
|
||||
capped[".agents/skills/check/SKILL.md"] = "---\ndescription: check totals\n---\nDo it.\n"
|
||||
self.assertEqual(autotrigger(capped), []) # 超上限时跳过，不硬塞
|
||||
|
||||
def test_paths_and_roundtrip(self):
|
||||
with self.assertRaises(ValueError):
|
||||
normalize_rel("../escape.py")
|
||||
self.assertFalse(is_allowed("/etc/passwd"))
|
||||
self.assertFalse(is_allowed("evil.py"))
|
||||
self.assertTrue(is_allowed(".agents/skills/a/SKILL.md"))
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
dest = Path(tmp) / "tree"
|
||||
write_tree(dict(BASE_TREE), dest)
|
||||
self.assertEqual(read_tree(dest), BASE_TREE)
|
||||
rejected = []
|
||||
(dest / "evil.py").write_text("x", encoding="utf-8")
|
||||
self.assertEqual(read_tree(dest, rejected=rejected), BASE_TREE)
|
||||
self.assertEqual(rejected, ["evil.py"])
|
||||
|
||||
def test_diff_and_diversity(self):
|
||||
changed = dict(BASE_TREE)
|
||||
changed["scripts/summarize.py"] = "changed"
|
||||
d = diff_summary(BASE_TREE, changed)
|
||||
self.assertEqual(d["modified"], ["scripts/summarize.py"])
|
||||
self.assertEqual(d["added"], [])
|
||||
div = diversity({"a": BASE_TREE, "b": changed})
|
||||
self.assertEqual(div["n_distinct"], 2)
|
||||
self.assertLess(div["mean_line_jaccard"], 1.0)
|
||||
|
||||
|
||||
class TestImprove(unittest.TestCase):
|
||||
def _run(self, mode, editor=None, selector=None, budget=None, k=1):
|
||||
tmp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(tmp.cleanup)
|
||||
root = Path(tmp.name)
|
||||
adapter = LedgerAdapter(root)
|
||||
library = Library(root / "library", "accounting")
|
||||
result = improve_task(
|
||||
adapter, editor or FixEditor(),
|
||||
selector=selector or PickSignedSelector(),
|
||||
reflector=LedgerReflector(), work_dir=root / "work",
|
||||
k=k, mode=mode, library=library, budget_usd=budget)
|
||||
return result, adapter, library, root
|
||||
|
||||
def test_parallel_k1_improves_and_enters_library(self):
|
||||
result, adapter, library, root = self._run("parallel")
|
||||
self.assertTrue(result.ok)
|
||||
self.assertEqual(result.selection.chosen_label, "agg1")
|
||||
chosen = result.mode_result.rollouts["agg1"]
|
||||
self.assertEqual(chosen.deliverable_text, "Total: 85.15")
|
||||
self.assertEqual(adapter.executions, 4) # base + inv1 + dom1 + agg1
|
||||
self.assertIsNotNone(result.library_entry)
|
||||
self.assertIn("ledger-1", library.entries())
|
||||
index = (root / "library" / "accounting" / "INDEX.md").read_text()
|
||||
self.assertIn("credits and debits", index)
|
||||
self.assertIn("85.15", result.selection.why + chosen.deliverable_text)
|
||||
|
||||
def test_sequential_k1_improves(self):
|
||||
result, adapter, library, root = self._run("sequential")
|
||||
self.assertTrue(result.ok)
|
||||
self.assertEqual(result.selection.chosen_label, "c1")
|
||||
self.assertEqual(result.mode_result.rollouts["c1"].deliverable_text, "Total: 85.15")
|
||||
self.assertEqual(adapter.executions, 4) # base + inv1 + dom1 + c1
|
||||
|
||||
def test_invalid_proposal_falls_back_to_base(self):
|
||||
result, adapter, library, root = self._run(
|
||||
"parallel", editor=BadEditor(), selector=PickFirstSelector())
|
||||
self.assertTrue(result.ok) # 回落的 base 产物仍可交付
|
||||
self.assertEqual(set(result.mode_result.fell_back), {"inv1", "dom1", "agg1"})
|
||||
self.assertEqual(result.mode_result.trees["agg1"], BASE_TREE)
|
||||
self.assertIsNone(result.library_entry) # 树没变，反思无经验，不入库
|
||||
|
||||
def test_budget_exceeded_blocks_library_but_still_delivers(self):
|
||||
result, adapter, library, root = self._run("parallel", budget=0.03)
|
||||
self.assertTrue(result.ok)
|
||||
self.assertTrue(result.budget_exceeded)
|
||||
self.assertIsNone(result.library_entry)
|
||||
|
||||
def test_k_validation(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
adapter = LedgerAdapter(Path(tmp))
|
||||
with self.assertRaises(ValueError):
|
||||
improve_task(adapter, FixEditor(), work_dir=Path(tmp) / "w", k=0)
|
||||
with self.assertRaises(ValueError):
|
||||
improve_task(adapter, FixEditor(), work_dir=Path(tmp) / "w", k=4)
|
||||
with self.assertRaises(ValueError):
|
||||
improve_task(adapter, FixEditor(), work_dir=Path(tmp) / "w", k=1,
|
||||
mode="sideways")
|
||||
|
||||
def test_trivial_k1_without_selector(self):
|
||||
tmp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(tmp.cleanup)
|
||||
root = Path(tmp.name)
|
||||
adapter = LedgerAdapter(root)
|
||||
result = improve_task(adapter, FixEditor(), selector=None,
|
||||
reflector=LedgerReflector(), work_dir=root / "work",
|
||||
k=1, mode="parallel")
|
||||
self.assertTrue(result.ok)
|
||||
self.assertTrue(result.selection.trivial)
|
||||
|
||||
|
||||
class TestSelection(unittest.TestCase):
|
||||
def test_drops_undelivered_and_falls_back_on_selector_error(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
adapter = LedgerAdapter(Path(tmp))
|
||||
rollouts = {
|
||||
"agg1": Rollout(label="agg1", ok=True, delivered=True,
|
||||
deliverable_text="Total: 1.00"),
|
||||
"agg2": Rollout(label="agg2", ok=False, delivered=False),
|
||||
}
|
||||
# k=2 的终版是 agg1/agg2；agg2 未交付应被丢掉，agg1 是唯一候选
|
||||
sel = select_final("ledger-1", 2, "parallel", rollouts, adapter=adapter,
|
||||
selector=RaisingSelector(), work_dir=Path(tmp) / "work",
|
||||
base_tree=dict(BASE_TREE))
|
||||
self.assertTrue(sel.ok)
|
||||
self.assertTrue(sel.fallback)
|
||||
self.assertEqual(sel.chosen_label, "agg1")
|
||||
self.assertIn("agg2", sel.dropped)
|
||||
|
||||
def test_assign_ids_is_stable(self):
|
||||
a = assign_ids(["agg2", "agg1"], task_id="t", k=1, mode="parallel")
|
||||
b = assign_ids(["agg1", "agg2"], task_id="t", k=1, mode="parallel")
|
||||
self.assertEqual(a, b) # 输入顺序不影响匿名映射
|
||||
|
||||
|
||||
class TestLibrary(unittest.TestCase):
|
||||
def test_add_requires_experience(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
lib = Library(Path(tmp) / "lib", "accounting")
|
||||
self.assertIsNone(lib.add("t1", dict(BASE_TREE), experience={}))
|
||||
self.assertEqual(lib.entries(), [])
|
||||
self.assertIsNone(lib.context_dir())
|
||||
entry = lib.add("t1", dict(BASE_TREE), why="why", label="agg1",
|
||||
experience={"cue": "c", "lesson": "l", "evidence": "e"})
|
||||
self.assertIsNotNone(entry)
|
||||
self.assertEqual(lib.entries(), ["t1"])
|
||||
self.assertEqual(lib.tree_of("t1"), BASE_TREE)
|
||||
self.assertIsNotNone(lib.context_dir())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,491 @@
|
||||
"""cloud/harness2_tasks.py 单测：真实 codex 任务版接线。
|
||||
|
||||
不跑真实模型、不联网：FakeTaskClient 在进程内模拟控制面——探针
|
||||
任务真跑 bundle 里的脚本，编辑任务按文件块格式回改动，选择与
|
||||
反思按最终回复格式回结果。HTTP 客户端则对本地假服务端实测。
|
||||
账本与 base 树同 test_harness2.py：带符号和 85.15，只加正数 105.25。
|
||||
"""
|
||||
|
||||
import csv
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import unittest
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
from cloud import harness2_tasks
|
||||
from cloud.harness2 import (
|
||||
parse_choice, parse_experience, parse_harness_files, parse_no_change,
|
||||
parse_notes, parse_why,
|
||||
)
|
||||
from cloud.harness2_tasks import (
|
||||
CodexEditor, CodexProbeAdapter, HttpTaskClient, TaskState,
|
||||
build_multipart, run_improvement_via_tasks, run_task,
|
||||
unwrap_result_text,
|
||||
)
|
||||
from cloud.uploads import parse_multipart, parse_paths_field
|
||||
except ImportError: # discover -s cloud/tests 时把项目根目录补进 sys.path
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||||
from cloud import harness2_tasks
|
||||
from cloud.harness2 import (
|
||||
parse_choice, parse_experience, parse_harness_files, parse_no_change,
|
||||
parse_notes, parse_why,
|
||||
)
|
||||
from cloud.harness2_tasks import (
|
||||
CodexEditor, CodexProbeAdapter, HttpTaskClient, TaskState,
|
||||
build_multipart, run_improvement_via_tasks, run_task,
|
||||
unwrap_result_text,
|
||||
)
|
||||
from cloud.uploads import parse_multipart, parse_paths_field
|
||||
|
||||
BASE_SCRIPT = """import csv
|
||||
import sys
|
||||
from decimal import Decimal
|
||||
with open(sys.argv[1]) as f:
|
||||
amounts = [Decimal(r['amount']) for r in csv.DictReader(f)]
|
||||
print(f"Total: {sum((a for a in amounts if a > 0), Decimal(0)):.2f}")
|
||||
"""
|
||||
|
||||
BASE_TREE = {
|
||||
"AGENTS.md": "Use scripts/summarize.py to total the ledger.\n",
|
||||
"scripts/summarize.py": BASE_SCRIPT,
|
||||
}
|
||||
|
||||
LEDGER = b"amount\n100.25\n-20.10\n5.00\n"
|
||||
|
||||
|
||||
class FakeTaskClient:
|
||||
"""进程内控制面：按 prompt 类型分派到确定性假 worker。"""
|
||||
|
||||
def __init__(self, editor_mode="fix", selector_mode="signed", envelope=False):
|
||||
self.editor_mode = editor_mode
|
||||
self.selector_mode = selector_mode
|
||||
self.envelope = envelope
|
||||
self.uploads = {}
|
||||
self.tasks = {}
|
||||
self.by_key = {}
|
||||
self.created = 0
|
||||
self.cancelled = []
|
||||
|
||||
def upload(self, files):
|
||||
uid = f"u{len(self.uploads) + 1}"
|
||||
self.uploads[uid] = dict(files)
|
||||
return uid
|
||||
|
||||
def create_task(self, *, project_id, prompt, upload_id=None,
|
||||
idempotency_key=None, **kwargs):
|
||||
if idempotency_key and idempotency_key in self.by_key:
|
||||
return self.by_key[idempotency_key]
|
||||
tid = f"t{len(self.tasks) + 1}"
|
||||
self.created += 1
|
||||
files = self.uploads.get(upload_id, {}) if upload_id else {}
|
||||
result = self._work(prompt, files)
|
||||
if self.envelope:
|
||||
# 模拟控制面真实形状：result_text 是 JSON 信封，text 字段
|
||||
# 里才是最终回复——接线层必须拆信封，否则行首锚定的文件
|
||||
# 块解析永远看不见块（2026-10-29 真实轮次根因）。
|
||||
result = json.dumps({
|
||||
"task": tid, "thread_id": "th", "cwd": "/x",
|
||||
"status": "completed", "text": result, "error": None,
|
||||
})
|
||||
self.tasks[tid] = TaskState(task_id=tid, status="completed",
|
||||
result_text=result)
|
||||
if idempotency_key:
|
||||
self.by_key[idempotency_key] = tid
|
||||
return tid
|
||||
|
||||
def _work(self, prompt, files):
|
||||
if prompt.startswith("You are editing an agent harness"):
|
||||
if self.editor_mode == "gibberish":
|
||||
return "hello, I did stuff"
|
||||
if self.editor_mode in ("notes-only-then-fix", "notes-only"):
|
||||
if ("YOUR PREVIOUS REPLY WAS REJECTED" in prompt
|
||||
and self.editor_mode == "notes-only-then-fix"):
|
||||
script = files["context/harness/scripts/summarize.py"].decode()
|
||||
fixed = script.replace(" if a > 0", "")
|
||||
return (
|
||||
"=== HARNESS FILE: scripts/summarize.py ===\n" + fixed +
|
||||
"=== END HARNESS FILE ===\n"
|
||||
"<notes>Removed the positive-only filter.</notes>")
|
||||
# 真实轮次原样复现：notes 里声称已修正，却不发文件块。
|
||||
return ("<notes>Removed the positive-only filter so signed "
|
||||
"amounts are summed; the script now totals all amounts."
|
||||
"</notes>")
|
||||
if self.editor_mode == "no-change":
|
||||
return ("<notes>No evidence justifies a change.</notes>"
|
||||
"<no-change>The base trajectory shows no defect."
|
||||
"</no-change>")
|
||||
script = files["context/harness/scripts/summarize.py"].decode()
|
||||
fixed = script.replace(" if a > 0", "")
|
||||
return (
|
||||
"=== HARNESS FILE: scripts/summarize.py ===\n" + fixed +
|
||||
"=== END HARNESS FILE ===\n"
|
||||
"<notes>Keep signed amounts: the base total omitted credits.</notes>"
|
||||
)
|
||||
if prompt.startswith("You are choosing which"):
|
||||
if self.selector_mode == "gibberish":
|
||||
return "I like them all"
|
||||
cands = sorted(p for p in files if p.startswith("context/candidates/")
|
||||
and p.endswith(".md"))
|
||||
for p in cands:
|
||||
if b"Total: 85.15" in files[p]:
|
||||
stem = Path(p).stem
|
||||
return (f"```choice\n{stem}\n```\n"
|
||||
"<why>The signed total matches the source ledger.</why>")
|
||||
stem = Path(cands[0]).stem
|
||||
return (f"```choice\n{stem}\n```\n<why>first delivered candidate</why>")
|
||||
if prompt.startswith("You are reading the complete record"):
|
||||
changed = (files.get("context/harness/scripts/summarize.py")
|
||||
!= files.get("context/base_harness/scripts/summarize.py"))
|
||||
if not changed:
|
||||
return "NO_ENTRY"
|
||||
return (
|
||||
"<edits>\nscripts/summarize.py: dropped the positive-only filter; "
|
||||
"verdict: helped\n</edits>\n"
|
||||
"<experience>\ncue: When a ledger includes credits and debits\n"
|
||||
"lesson: Include both signs when summing in scripts/summarize.py.\n"
|
||||
"evidence: Removing the positive-only filter changed the probe total "
|
||||
"to include the credits omitted by the base trajectory.\n"
|
||||
"confidence: high\n</experience>"
|
||||
)
|
||||
# 探针：真跑 bundle 里的脚本对 bundle 里的账本求和。
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
for rel, data in files.items():
|
||||
dest = root / rel
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
dest.write_bytes(data)
|
||||
proc = subprocess.run(
|
||||
[sys.executable, str(root / "scripts" / "summarize.py"),
|
||||
str(root / "ledger.csv")],
|
||||
capture_output=True, text=True, timeout=10)
|
||||
return proc.stdout.strip()
|
||||
|
||||
def get_task(self, task_id):
|
||||
return self.tasks[task_id]
|
||||
|
||||
def get_trace_text(self, task_id):
|
||||
return "[ACTION] fake probe\n[OBSERVATION]\n" + self.tasks[task_id].result_text
|
||||
|
||||
def get_cost_usd(self, task_id):
|
||||
return 0.02
|
||||
|
||||
def cancel_task(self, task_id):
|
||||
self.cancelled.append(task_id)
|
||||
|
||||
|
||||
class TestParseHelpers(unittest.TestCase):
|
||||
def test_multipart_roundtrips_through_server_parser(self):
|
||||
files = {"context/task.md": b"do it", "AGENTS.md": b"be nice\n"}
|
||||
body, content_type = build_multipart(files)
|
||||
fields, parsed = parse_multipart(body, content_type)
|
||||
paths = parse_paths_field(fields, parsed)
|
||||
self.assertEqual(paths, sorted(files))
|
||||
self.assertEqual(dict(parsed), files)
|
||||
|
||||
def test_choice_why_notes_experience(self):
|
||||
self.assertEqual(parse_choice("```choice\nB\n```"), "B")
|
||||
self.assertIsNone(parse_choice("no choice here"))
|
||||
self.assertEqual(parse_why("<why>because</why>"), "because")
|
||||
self.assertEqual(parse_notes("<notes>did x</notes>"), "did x")
|
||||
exp = parse_experience(
|
||||
"<experience>\ncue: c\nlesson: l\nevidence: e\nconfidence: high\n</experience>")
|
||||
self.assertEqual((exp["cue"], exp["lesson"], exp["evidence"]), ("c", "l", "e"))
|
||||
self.assertEqual(parse_experience("NO_ENTRY"), {})
|
||||
self.assertEqual(parse_experience("<experience>\ncue: c\n</experience>"), {})
|
||||
|
||||
def test_harness_file_blocks(self):
|
||||
text = ("=== HARNESS FILE: AGENTS.md ===\nhello\n"
|
||||
"=== END HARNESS FILE ===\n<notes>n</notes>")
|
||||
self.assertEqual(parse_harness_files(text), {"AGENTS.md": "hello\n"})
|
||||
self.assertEqual(parse_harness_files("nothing here"), {})
|
||||
with self.assertRaises(ValueError):
|
||||
parse_harness_files("=== HARNESS FILE: /etc/passwd ===\nx\n"
|
||||
"=== END HARNESS FILE ===")
|
||||
with self.assertRaises(ValueError):
|
||||
parse_harness_files("=== HARNESS FILE: AGENTS.md ===\nunterminated\n")
|
||||
|
||||
|
||||
class TestFullRunViaTasks(unittest.TestCase):
|
||||
def _run(self, client, run_tag="run1", work_sub="work"):
|
||||
tmp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(tmp.cleanup)
|
||||
root = Path(tmp.name)
|
||||
result = run_improvement_via_tasks(
|
||||
client, project_id="p1", task_id="ledger-1",
|
||||
prompt="Sum every signed amount in ledger.csv. Return Total: with two decimals.",
|
||||
base_harness=dict(BASE_TREE), run_tag=run_tag,
|
||||
work_dir=root / work_sub, domain="accounting",
|
||||
source_files={"ledger.csv": LEDGER},
|
||||
k=1, mode="parallel", library_root=root / "library")
|
||||
return result, client, root
|
||||
|
||||
def test_parallel_k1_via_real_task_shape(self):
|
||||
client = FakeTaskClient()
|
||||
result, _, root = self._run(client)
|
||||
self.assertTrue(result.ok)
|
||||
self.assertEqual(result.selection.chosen_label, "agg1")
|
||||
self.assertEqual(result.mode_result.rollouts["agg1"].deliverable_text,
|
||||
"Total: 85.15")
|
||||
# 4 探针 + 3 编辑 + 1 选择 + 1 反思 = 9 个任务
|
||||
self.assertEqual(client.created, 9)
|
||||
self.assertIsNotNone(result.library_entry)
|
||||
self.assertTrue((root / "library" / "accounting" / "INDEX.md").is_file())
|
||||
|
||||
def test_same_run_tag_replays_without_new_tasks(self):
|
||||
client = FakeTaskClient()
|
||||
result1, _, _ = self._run(client, work_sub="work1")
|
||||
self.assertTrue(result1.ok)
|
||||
created_after_first = client.created
|
||||
result2, _, _ = self._run(client, work_sub="work2")
|
||||
self.assertTrue(result2.ok)
|
||||
self.assertEqual(client.created, created_after_first) # 幂等键命中，零新建
|
||||
|
||||
def test_editor_gibberish_falls_back(self):
|
||||
client = FakeTaskClient(editor_mode="gibberish")
|
||||
result, _, _ = self._run(client)
|
||||
self.assertTrue(result.ok) # 回落的 base 产物仍可交付
|
||||
self.assertEqual(set(result.mode_result.fell_back), {"inv1", "dom1", "agg1"})
|
||||
self.assertIsNone(result.library_entry)
|
||||
|
||||
def test_selector_gibberish_falls_back(self):
|
||||
client = FakeTaskClient(selector_mode="gibberish")
|
||||
result, _, _ = self._run(client)
|
||||
self.assertTrue(result.ok)
|
||||
self.assertTrue(result.selection.fallback)
|
||||
|
||||
|
||||
class TestEditorFormatRetry(unittest.TestCase):
|
||||
"""编辑格式纠正：notes-only 先重试再落盘；显式无改动一次通过；
|
||||
两次都 notes-only 则如实抛错（阶段回落，不静默采纳）。"""
|
||||
|
||||
def _editor(self, mode):
|
||||
tmp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(tmp.cleanup)
|
||||
ctx = Path(tmp.name)
|
||||
(ctx / "harness" / "scripts").mkdir(parents=True)
|
||||
for rel, content in BASE_TREE.items():
|
||||
(ctx / "harness" / rel).write_text(content)
|
||||
client = FakeTaskClient(editor_mode=mode)
|
||||
editor = CodexEditor(client, project_id="p", run_tag="fmt")
|
||||
return editor, ctx, client
|
||||
|
||||
def test_notes_only_then_fix_lands(self):
|
||||
editor, ctx, client = self._editor("notes-only-then-fix")
|
||||
result = editor.edit("You are editing an agent harness", ctx, "inv1.a1")
|
||||
self.assertNotIn("if a > 0", result.tree["scripts/summarize.py"])
|
||||
self.assertEqual(client.created, 2) # 首发 + 一次格式纠正
|
||||
|
||||
def test_no_change_marker_keeps_base(self):
|
||||
editor, ctx, client = self._editor("no-change")
|
||||
result = editor.edit("You are editing an agent harness", ctx, "inv1.a1")
|
||||
self.assertEqual(result.tree["scripts/summarize.py"], BASE_SCRIPT)
|
||||
self.assertEqual(client.created, 1)
|
||||
|
||||
def test_notes_only_twice_raises(self):
|
||||
editor, ctx, client = self._editor("notes-only")
|
||||
with self.assertRaises(ValueError):
|
||||
editor.edit("You are editing an agent harness", ctx, "inv1.a1")
|
||||
self.assertEqual(client.created, 2)
|
||||
|
||||
def test_parse_no_change(self):
|
||||
self.assertEqual(parse_no_change("<no-change>no defect</no-change>"),
|
||||
"no defect")
|
||||
self.assertEqual(parse_no_change("just notes"), "")
|
||||
|
||||
|
||||
class TestUnwrapResultText(unittest.TestCase):
|
||||
"""拆信封：信封取 text；其它原样返回，不丢内容。"""
|
||||
|
||||
def test_envelope_unwrapped(self):
|
||||
raw = json.dumps({"task": "t1", "status": "completed",
|
||||
"text": "=== HARNESS FILE: a.md ===\nhi\n=== END HARNESS FILE ===\n"})
|
||||
self.assertTrue(unwrap_result_text(raw).startswith("=== HARNESS FILE"))
|
||||
|
||||
def test_plain_text_unchanged(self):
|
||||
self.assertEqual(unwrap_result_text("Total: 85.15"), "Total: 85.15")
|
||||
self.assertEqual(unwrap_result_text('{"no": "text field"}'),
|
||||
'{"no": "text field"}')
|
||||
self.assertEqual(unwrap_result_text("{not json"), "{not json")
|
||||
self.assertEqual(unwrap_result_text(""), "")
|
||||
|
||||
|
||||
class TestEnvelopeFullRound(unittest.TestCase):
|
||||
"""信封形状整轮：假控制面按真实信封返回时，编辑块仍能落盘、
|
||||
探针交付物是干净文本（而非信封 JSON），经验正常入库。"""
|
||||
|
||||
def test_envelope_round(self):
|
||||
client = FakeTaskClient(envelope=True)
|
||||
tmp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(tmp.cleanup)
|
||||
root = Path(tmp.name)
|
||||
result = run_improvement_via_tasks(
|
||||
client, project_id="p1", task_id="ledger-1",
|
||||
prompt="Sum every signed amount in ledger.csv. Return Total: with two decimals.",
|
||||
base_harness=dict(BASE_TREE), run_tag="env",
|
||||
work_dir=root / "work", domain="accounting",
|
||||
source_files={"ledger.csv": LEDGER},
|
||||
k=1, mode="parallel", library_root=root / "library")
|
||||
self.assertTrue(result.ok)
|
||||
self.assertEqual(result.mode_result.rollouts["agg1"].deliverable_text,
|
||||
"Total: 85.15")
|
||||
self.assertNotIn("thread_id",
|
||||
result.mode_result.rollouts["agg1"].deliverable_text)
|
||||
self.assertIsNotNone(result.library_entry)
|
||||
|
||||
|
||||
class TestHttpClient(unittest.TestCase):
|
||||
def test_client_against_local_server(self):
|
||||
seen = {"keys": [], "uploads": [], "tasks": []}
|
||||
polls = {"t1": 0}
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
def log_message(self, *args):
|
||||
pass
|
||||
|
||||
def _json(self, status, obj):
|
||||
raw = json.dumps(obj).encode()
|
||||
self.send_response(status)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(raw)))
|
||||
self.end_headers()
|
||||
self.wfile.write(raw)
|
||||
|
||||
def do_POST(self):
|
||||
seen["keys"].append(self.headers.get("X-API-Key"))
|
||||
length = int(self.headers.get("Content-Length") or 0)
|
||||
body = self.rfile.read(length) if length else b""
|
||||
if self.path == "/uploads":
|
||||
seen["uploads"].append(body)
|
||||
self._json(201, {"upload_id": "u1", "files": []})
|
||||
elif self.path == "/tasks":
|
||||
seen["tasks"].append(json.loads(body.decode()))
|
||||
self._json(201, {"id": "t1"})
|
||||
elif self.path == "/tasks/t1/cancel":
|
||||
self._json(200, {})
|
||||
else:
|
||||
self._json(404, {"error": "not found"})
|
||||
|
||||
def do_GET(self):
|
||||
if self.path == "/tasks/t1":
|
||||
polls["t1"] += 1
|
||||
status = "queued" if polls["t1"] == 1 else "completed"
|
||||
self._json(200, {"id": "t1", "status": status,
|
||||
"result_text": "Total: 85.15" if status == "completed" else "",
|
||||
"error_text": ""})
|
||||
elif self.path == "/tasks/t1/cost":
|
||||
self._json(200, {"task_id": "t1", "usd_micros": 2500000})
|
||||
elif self.path == "/tasks/t1/trace":
|
||||
self._json(200, {"entries": [{"id": "e1", "kind": "agent",
|
||||
"text": "hello"}]})
|
||||
else:
|
||||
self._json(404, {"error": "not found"})
|
||||
|
||||
server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
|
||||
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
try:
|
||||
client = HttpTaskClient(f"http://127.0.0.1:{server.server_address[1]}",
|
||||
"test-key")
|
||||
done = run_task(client, project_id="p1", prompt="do it",
|
||||
files={"a.txt": b"hi"}, poll_seconds=0.01,
|
||||
timeout_seconds=10)
|
||||
finally:
|
||||
server.shutdown()
|
||||
server.server_close()
|
||||
self.assertEqual(done.state.status, "completed")
|
||||
self.assertEqual(done.state.result_text, "Total: 85.15")
|
||||
self.assertEqual(done.cost_usd, 2.5)
|
||||
self.assertIn("hello", done.trace_text)
|
||||
self.assertTrue(all(k == "test-key" for k in seen["keys"]))
|
||||
self.assertEqual(seen["tasks"][0]["project_id"], "p1")
|
||||
fields, files = parse_multipart(seen["uploads"][0],
|
||||
"multipart/form-data; boundary=" +
|
||||
seen["uploads"][0].split(b"\r\n")[0][2:].decode())
|
||||
self.assertEqual(dict(files), {"a.txt": b"hi"})
|
||||
|
||||
|
||||
class TestHttpClientRetry(unittest.TestCase):
|
||||
"""瞬断重试：掉连接与 503 各重试一次后成功；401 只请求一次。"""
|
||||
|
||||
def _serve(self, handler_cls):
|
||||
server = ThreadingHTTPServer(("127.0.0.1", 0), handler_cls)
|
||||
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
self.addCleanup(server.shutdown)
|
||||
self.addCleanup(server.server_close)
|
||||
return HttpTaskClient(
|
||||
f"http://127.0.0.1:{server.server_address[1]}", "test-key",
|
||||
timeout=5)
|
||||
|
||||
def test_drop_then_success(self):
|
||||
counts = {"n": 0}
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
def log_message(self, *args):
|
||||
pass
|
||||
|
||||
def do_GET(self):
|
||||
counts["n"] += 1
|
||||
if counts["n"] == 1:
|
||||
self.close_connection = True
|
||||
return
|
||||
raw = json.dumps({"ok": True}).encode()
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(raw)))
|
||||
self.end_headers()
|
||||
self.wfile.write(raw)
|
||||
|
||||
client = self._serve(Handler)
|
||||
self.assertEqual(client._request("GET", "/flaky"), {"ok": True})
|
||||
self.assertEqual(counts["n"], 2)
|
||||
|
||||
def test_503_then_success(self):
|
||||
counts = {"n": 0}
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
def log_message(self, *args):
|
||||
pass
|
||||
|
||||
def do_GET(self):
|
||||
counts["n"] += 1
|
||||
raw = json.dumps({"ok": counts["n"] > 1}).encode()
|
||||
self.send_response(503 if counts["n"] == 1 else 200)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(raw)))
|
||||
self.end_headers()
|
||||
self.wfile.write(raw)
|
||||
|
||||
client = self._serve(Handler)
|
||||
self.assertEqual(client._request("GET", "/flaky"), {"ok": True})
|
||||
self.assertEqual(counts["n"], 2)
|
||||
|
||||
def test_401_not_retried(self):
|
||||
counts = {"n": 0}
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
def log_message(self, *args):
|
||||
pass
|
||||
|
||||
def do_GET(self):
|
||||
counts["n"] += 1
|
||||
raw = json.dumps({"error": "unauthorized"}).encode()
|
||||
self.send_response(401)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(raw)))
|
||||
self.end_headers()
|
||||
self.wfile.write(raw)
|
||||
|
||||
client = self._serve(Handler)
|
||||
with self.assertRaises(RuntimeError):
|
||||
client._request("GET", "/flaky")
|
||||
self.assertEqual(counts["n"], 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user