host
42abe3cc5b
Port Google Research's Harness^2 method layer (model weights frozen, only the surrounding harness is optimized): Propose-Probe-Compose with INV/DOM/AGG editors, parallel/sequential modes, per-domain experience library, anonymous-product selection, k<=3 hard cap and budget gate. Execution goes only through the existing control-plane API (upload -> task with idempotency key -> poll -> trace/cost -> cancel on timeout); no new server endpoints, no codex flags composed, tasks inherit the normal executor/sandbox/quota/audit path. Wiring lessons from four real rounds against production (project wwww, ledger task): HttpTaskClient retries transient disconnects and 502/503/429 (3 attempts); editors get a worked file-block example, an explicit <no-change> marker and one feedback retry (.fmt2) instead of silently accepting notes-only replies; run_task unwraps the server's result_text JSON envelope (text field holds the real reply) — without this, line-anchored file-block parsing could never see blocks the agents were actually emitting. Final round (real4) fully green: all stages landed, selection non-fallback, experience入库, probe delivers the correct signed total 85.15. Tests: 35 harness tests (17 method + 18 wiring incl. fake-control-plane full round, idempotent replay, retry, format retry, envelope round). Deliberate deviations from upstream are documented in docs/harness2.md (no network-unrestricted/sandbox-disabled defaults, auto_apply off, no benchmark/scorer, probe deliverable = final reply text).
336 lines
14 KiB
Python
336 lines
14 KiB
Python
"""cloud/harness2.py 单测：递归改进循环。
|
||
|
||
不跑真实模型、不联网：执行器在进程内跑 harness 树里的脚本，
|
||
编辑器/选择器/反思器都是确定性假实现（上游 harness2-demo 的同款
|
||
做法：base 脚本只加正数，编辑器去掉正数过滤，正确总数是带符号和）。
|
||
|
||
账本：100.25, -20.10, 5.00 —— 带符号和 85.15，只加正数是 105.25。
|
||
"""
|
||
|
||
import csv
|
||
import subprocess
|
||
import sys
|
||
import tempfile
|
||
import unittest
|
||
from pathlib import Path
|
||
from types import SimpleNamespace
|
||
|
||
try:
|
||
from cloud import harness2
|
||
from cloud.harness2 import (
|
||
Library, Rollout, EditResult, Budget, build_context, validate,
|
||
read_tree, write_tree, is_allowed, normalize_rel, autotrigger,
|
||
skill_names, diff_summary, diversity, improve_task, select_final,
|
||
assign_ids, has_experience,
|
||
)
|
||
except ImportError: # discover -s cloud/tests 时把项目根目录补进 sys.path
|
||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||
from cloud import harness2
|
||
from cloud.harness2 import (
|
||
Library, Rollout, EditResult, Budget, build_context, validate,
|
||
read_tree, write_tree, is_allowed, normalize_rel, autotrigger,
|
||
skill_names, diff_summary, diversity, improve_task, select_final,
|
||
assign_ids, has_experience,
|
||
)
|
||
|
||
BASE_SCRIPT = """import csv
|
||
import sys
|
||
from decimal import Decimal
|
||
with open(sys.argv[1]) as f:
|
||
amounts = [Decimal(r['amount']) for r in csv.DictReader(f)]
|
||
print(f"Total: {sum((a for a in amounts if a > 0), Decimal(0)):.2f}")
|
||
"""
|
||
|
||
BASE_TREE = {
|
||
"AGENTS.md": "Use scripts/summarize.py to total the ledger.\n",
|
||
"scripts/summarize.py": BASE_SCRIPT,
|
||
}
|
||
|
||
|
||
class LedgerAdapter:
|
||
"""任务：对账本求和。执行 = 真跑 harness 树里的脚本（无模型）。"""
|
||
|
||
task_id = "ledger-1"
|
||
domain = "accounting"
|
||
prompt = "Sum every signed amount in ledger.csv. Return Total: with two decimals."
|
||
|
||
def __init__(self, root):
|
||
self.root = Path(root)
|
||
self.executions = 0
|
||
self.ledger = self.root / "ledger.csv"
|
||
self.ledger.write_text("amount\n100.25\n-20.10\n5.00\n", encoding="utf-8")
|
||
|
||
def base_harness(self):
|
||
return dict(BASE_TREE)
|
||
|
||
def execute(self, label, harness):
|
||
self.executions += 1
|
||
work = self.root / "runs" / label
|
||
work.mkdir(parents=True, exist_ok=True)
|
||
write_tree(harness, work / "harness")
|
||
proc = subprocess.run(
|
||
[sys.executable, str(work / "harness" / "scripts" / "summarize.py"),
|
||
str(self.ledger)],
|
||
capture_output=True, text=True, timeout=10,
|
||
)
|
||
out = proc.stdout.strip()
|
||
return Rollout(
|
||
label=label, ok=proc.returncode == 0, delivered=out.startswith("Total:"),
|
||
deliverable_text=out, trajectory="[ACTION] Run scripts/summarize.py\n"
|
||
"[OBSERVATION]\n" + out, cost_usd=0.01,
|
||
error="" if proc.returncode == 0 else proc.stderr[-200:],
|
||
)
|
||
|
||
def validate_tree(self, harness):
|
||
if "scripts/summarize.py" not in harness:
|
||
raise ValueError("The local executor requires scripts/summarize.py")
|
||
|
||
|
||
class FixEditor:
|
||
"""确定性编辑器：去掉正数过滤；同时在 context/harness/ 里真实改文件，
|
||
模拟 codex 任务编辑工作目录的方式，再把改完的树读回。"""
|
||
|
||
def edit(self, mission, context_dir, label):
|
||
harness_dir = Path(context_dir) / "harness"
|
||
script = harness_dir / "scripts" / "summarize.py"
|
||
script.write_text(script.read_text().replace(" if a > 0", ""), encoding="utf-8")
|
||
return EditResult(tree=read_tree(harness_dir),
|
||
notes="Keep signed amounts: the base total omitted credits.")
|
||
|
||
|
||
class BadEditor:
|
||
"""永远交回非法树：AGENTS.md 为空，校验必拒，最终必须回落到 base。"""
|
||
|
||
def edit(self, mission, context_dir, label):
|
||
return EditResult(tree={"AGENTS.md": ""}, notes="bad proposal")
|
||
|
||
|
||
class PickSignedSelector:
|
||
def select(self, mission, context_dir):
|
||
for path in sorted((Path(context_dir) / "candidates").glob("*.md")):
|
||
if "Total: 85.15" in path.read_text():
|
||
return path.stem, "The signed total matches the source ledger.", 0.0
|
||
raise ValueError("no candidate with the signed total")
|
||
|
||
|
||
class PickFirstSelector:
|
||
def select(self, mission, context_dir):
|
||
paths = sorted((Path(context_dir) / "candidates").glob("*.md"))
|
||
return paths[0].stem, "first delivered candidate", 0.0
|
||
|
||
|
||
class RaisingSelector:
|
||
def select(self, mission, context_dir):
|
||
raise RuntimeError("selector session failed")
|
||
|
||
|
||
class LedgerReflector:
|
||
def reflect(self, mission, context_dir):
|
||
root = Path(context_dir)
|
||
changed = (root / "harness" / "scripts" / "summarize.py").read_text() != (
|
||
root / "base_harness" / "scripts" / "summarize.py").read_text()
|
||
if not changed:
|
||
return {}
|
||
return {
|
||
"cue": "When a ledger includes credits and debits",
|
||
"lesson": "Include both signs when summing in scripts/summarize.py.",
|
||
"evidence": "Removing the positive-only filter changed the probe total "
|
||
"to include the credits omitted by the base trajectory.",
|
||
"confidence": "high for this observed arithmetic contrast",
|
||
"edits": "scripts/summarize.py: dropped the positive-only filter; observed: "
|
||
"probe total included the omitted credits; verdict: helped",
|
||
}
|
||
|
||
|
||
class TestTree(unittest.TestCase):
|
||
def test_validate_accepts_base(self):
|
||
census = validate(dict(BASE_TREE))
|
||
self.assertEqual(census["scripts"], 1)
|
||
self.assertEqual(census["n_files"], 2)
|
||
|
||
def test_validate_rejects_skill_without_description(self):
|
||
tree = dict(BASE_TREE)
|
||
tree[".agents/skills/check/SKILL.md"] = "Do the check.\n"
|
||
tree["AGENTS.md"] += "- When checking, read .agents/skills/check/SKILL.md.\n"
|
||
with self.assertRaises(ValueError) as cm:
|
||
validate(tree)
|
||
self.assertIn("description", str(cm.exception))
|
||
|
||
def test_validate_rejects_untriggered_skill(self):
|
||
tree = dict(BASE_TREE)
|
||
tree[".agents/skills/check/SKILL.md"] = "---\ndescription: check totals\n---\nDo it.\n"
|
||
with self.assertRaises(ValueError) as cm:
|
||
validate(tree)
|
||
self.assertIn("never named", str(cm.exception))
|
||
|
||
def test_validate_rejects_bad_hooks_and_bad_script(self):
|
||
tree = dict(BASE_TREE)
|
||
tree[".codex/hooks.json"] = '{"hooks": {"Nope": []}}'
|
||
with self.assertRaises(ValueError):
|
||
validate(tree)
|
||
tree = dict(BASE_TREE)
|
||
tree["scripts/summarize.py"] = "def broken(:\n"
|
||
with self.assertRaises(ValueError):
|
||
validate(tree)
|
||
tree = dict(BASE_TREE)
|
||
tree["AGENTS.md"] = "x" * 8001
|
||
with self.assertRaises(ValueError):
|
||
validate(tree)
|
||
|
||
def test_validate_rejects_missing_script_ref(self):
|
||
tree = dict(BASE_TREE)
|
||
tree["AGENTS.md"] += "Run python3 scripts/nope.py first.\n"
|
||
with self.assertRaises(ValueError):
|
||
validate(tree)
|
||
|
||
def test_autotrigger_repairs_and_respects_cap(self):
|
||
tree = dict(BASE_TREE)
|
||
tree[".agents/skills/check/SKILL.md"] = "---\ndescription: check totals\n---\nDo it.\n"
|
||
fixed = autotrigger(tree)
|
||
self.assertEqual(fixed, ["check"])
|
||
self.assertIn("check", tree["AGENTS.md"])
|
||
validate(tree) # 补完触发行后应能通过
|
||
capped = dict(BASE_TREE)
|
||
capped["AGENTS.md"] = "x" * 7995
|
||
capped[".agents/skills/check/SKILL.md"] = "---\ndescription: check totals\n---\nDo it.\n"
|
||
self.assertEqual(autotrigger(capped), []) # 超上限时跳过，不硬塞
|
||
|
||
def test_paths_and_roundtrip(self):
|
||
with self.assertRaises(ValueError):
|
||
normalize_rel("../escape.py")
|
||
self.assertFalse(is_allowed("/etc/passwd"))
|
||
self.assertFalse(is_allowed("evil.py"))
|
||
self.assertTrue(is_allowed(".agents/skills/a/SKILL.md"))
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
dest = Path(tmp) / "tree"
|
||
write_tree(dict(BASE_TREE), dest)
|
||
self.assertEqual(read_tree(dest), BASE_TREE)
|
||
rejected = []
|
||
(dest / "evil.py").write_text("x", encoding="utf-8")
|
||
self.assertEqual(read_tree(dest, rejected=rejected), BASE_TREE)
|
||
self.assertEqual(rejected, ["evil.py"])
|
||
|
||
def test_diff_and_diversity(self):
|
||
changed = dict(BASE_TREE)
|
||
changed["scripts/summarize.py"] = "changed"
|
||
d = diff_summary(BASE_TREE, changed)
|
||
self.assertEqual(d["modified"], ["scripts/summarize.py"])
|
||
self.assertEqual(d["added"], [])
|
||
div = diversity({"a": BASE_TREE, "b": changed})
|
||
self.assertEqual(div["n_distinct"], 2)
|
||
self.assertLess(div["mean_line_jaccard"], 1.0)
|
||
|
||
|
||
class TestImprove(unittest.TestCase):
|
||
def _run(self, mode, editor=None, selector=None, budget=None, k=1):
|
||
tmp = tempfile.TemporaryDirectory()
|
||
self.addCleanup(tmp.cleanup)
|
||
root = Path(tmp.name)
|
||
adapter = LedgerAdapter(root)
|
||
library = Library(root / "library", "accounting")
|
||
result = improve_task(
|
||
adapter, editor or FixEditor(),
|
||
selector=selector or PickSignedSelector(),
|
||
reflector=LedgerReflector(), work_dir=root / "work",
|
||
k=k, mode=mode, library=library, budget_usd=budget)
|
||
return result, adapter, library, root
|
||
|
||
def test_parallel_k1_improves_and_enters_library(self):
|
||
result, adapter, library, root = self._run("parallel")
|
||
self.assertTrue(result.ok)
|
||
self.assertEqual(result.selection.chosen_label, "agg1")
|
||
chosen = result.mode_result.rollouts["agg1"]
|
||
self.assertEqual(chosen.deliverable_text, "Total: 85.15")
|
||
self.assertEqual(adapter.executions, 4) # base + inv1 + dom1 + agg1
|
||
self.assertIsNotNone(result.library_entry)
|
||
self.assertIn("ledger-1", library.entries())
|
||
index = (root / "library" / "accounting" / "INDEX.md").read_text()
|
||
self.assertIn("credits and debits", index)
|
||
self.assertIn("85.15", result.selection.why + chosen.deliverable_text)
|
||
|
||
def test_sequential_k1_improves(self):
|
||
result, adapter, library, root = self._run("sequential")
|
||
self.assertTrue(result.ok)
|
||
self.assertEqual(result.selection.chosen_label, "c1")
|
||
self.assertEqual(result.mode_result.rollouts["c1"].deliverable_text, "Total: 85.15")
|
||
self.assertEqual(adapter.executions, 4) # base + inv1 + dom1 + c1
|
||
|
||
def test_invalid_proposal_falls_back_to_base(self):
|
||
result, adapter, library, root = self._run(
|
||
"parallel", editor=BadEditor(), selector=PickFirstSelector())
|
||
self.assertTrue(result.ok) # 回落的 base 产物仍可交付
|
||
self.assertEqual(set(result.mode_result.fell_back), {"inv1", "dom1", "agg1"})
|
||
self.assertEqual(result.mode_result.trees["agg1"], BASE_TREE)
|
||
self.assertIsNone(result.library_entry) # 树没变，反思无经验，不入库
|
||
|
||
def test_budget_exceeded_blocks_library_but_still_delivers(self):
|
||
result, adapter, library, root = self._run("parallel", budget=0.03)
|
||
self.assertTrue(result.ok)
|
||
self.assertTrue(result.budget_exceeded)
|
||
self.assertIsNone(result.library_entry)
|
||
|
||
def test_k_validation(self):
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
adapter = LedgerAdapter(Path(tmp))
|
||
with self.assertRaises(ValueError):
|
||
improve_task(adapter, FixEditor(), work_dir=Path(tmp) / "w", k=0)
|
||
with self.assertRaises(ValueError):
|
||
improve_task(adapter, FixEditor(), work_dir=Path(tmp) / "w", k=4)
|
||
with self.assertRaises(ValueError):
|
||
improve_task(adapter, FixEditor(), work_dir=Path(tmp) / "w", k=1,
|
||
mode="sideways")
|
||
|
||
def test_trivial_k1_without_selector(self):
|
||
tmp = tempfile.TemporaryDirectory()
|
||
self.addCleanup(tmp.cleanup)
|
||
root = Path(tmp.name)
|
||
adapter = LedgerAdapter(root)
|
||
result = improve_task(adapter, FixEditor(), selector=None,
|
||
reflector=LedgerReflector(), work_dir=root / "work",
|
||
k=1, mode="parallel")
|
||
self.assertTrue(result.ok)
|
||
self.assertTrue(result.selection.trivial)
|
||
|
||
|
||
class TestSelection(unittest.TestCase):
|
||
def test_drops_undelivered_and_falls_back_on_selector_error(self):
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
adapter = LedgerAdapter(Path(tmp))
|
||
rollouts = {
|
||
"agg1": Rollout(label="agg1", ok=True, delivered=True,
|
||
deliverable_text="Total: 1.00"),
|
||
"agg2": Rollout(label="agg2", ok=False, delivered=False),
|
||
}
|
||
# k=2 的终版是 agg1/agg2；agg2 未交付应被丢掉，agg1 是唯一候选
|
||
sel = select_final("ledger-1", 2, "parallel", rollouts, adapter=adapter,
|
||
selector=RaisingSelector(), work_dir=Path(tmp) / "work",
|
||
base_tree=dict(BASE_TREE))
|
||
self.assertTrue(sel.ok)
|
||
self.assertTrue(sel.fallback)
|
||
self.assertEqual(sel.chosen_label, "agg1")
|
||
self.assertIn("agg2", sel.dropped)
|
||
|
||
def test_assign_ids_is_stable(self):
|
||
a = assign_ids(["agg2", "agg1"], task_id="t", k=1, mode="parallel")
|
||
b = assign_ids(["agg1", "agg2"], task_id="t", k=1, mode="parallel")
|
||
self.assertEqual(a, b) # 输入顺序不影响匿名映射
|
||
|
||
|
||
class TestLibrary(unittest.TestCase):
|
||
def test_add_requires_experience(self):
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
lib = Library(Path(tmp) / "lib", "accounting")
|
||
self.assertIsNone(lib.add("t1", dict(BASE_TREE), experience={}))
|
||
self.assertEqual(lib.entries(), [])
|
||
self.assertIsNone(lib.context_dir())
|
||
entry = lib.add("t1", dict(BASE_TREE), why="why", label="agg1",
|
||
experience={"cue": "c", "lesson": "l", "evidence": "e"})
|
||
self.assertIsNotNone(entry)
|
||
self.assertEqual(lib.entries(), ["t1"])
|
||
self.assertEqual(lib.tree_of("t1"), BASE_TREE)
|
||
self.assertIsNotNone(lib.context_dir())
|
||
|
||
|
||
if __name__ == "__main__":
|
||
unittest.main()
|