harness2: upstream gap fill — trivial selection fix, session budget, run_tag validation, preflight check

- select_final trivial condition now k==1 and single-candidate pool,
  regardless of whether a selector is provided (previously gated on
  selector is None, so wired runs always paid for a useless selector
  task at k=1)
- Budget gains max_sessions gate (probe + stage-attempt sessions;
  fmt2 corrections and selector/reflector sessions exempt), the
  backstop for unpriced models where the USD gate never binds
- validate_run_tag (charset, no leading '-', no s<N> seed prefix)
  enforced at run_improvement_via_tasks and the CLI
- CLI --check preflight (base harness, source dir, prompt, tag,
  control-plane /health probe) without creating tasks
- ModeResult.to_dict now includes rollout summaries so a lost probe
  is visible in full_result instead of only on the server
- CLI plan counts the selector only when k>1
This commit is contained in:
host committed 2026-10-29 15:31:58 +08:00
1 parent 3d60f32bd3
commit 4ed1479d7a
5 files changed
+247 -26

No files matched your search

+85 -9
View File
@@ -24,8 +24,8 @@ try:
)
from cloud.harness2_tasks import (
CodexEditor, CodexProbeAdapter, HttpTaskClient, TaskState,
build_multipart, run_improvement_via_tasks, run_task,
unwrap_result_text,
build_multipart, preflight_checks, run_improvement_via_tasks, run_task,
unwrap_result_text, validate_run_tag,
)
from cloud.uploads import parse_multipart, parse_paths_field
except ImportError: # discover -s cloud/tests 时把项目根目录补进 sys.path
@@ -37,8 +37,8 @@ except ImportError: # discover -s cloud/tests 时把项目根目录补进 sys.p
)
from cloud.harness2_tasks import (
CodexEditor, CodexProbeAdapter, HttpTaskClient, TaskState,
build_multipart, run_improvement_via_tasks, run_task,
unwrap_result_text,
build_multipart, preflight_checks, run_improvement_via_tasks, run_task,
unwrap_result_text, validate_run_tag,
)
from cloud.uploads import parse_multipart, parse_paths_field
@@ -211,7 +211,7 @@ class TestParseHelpers(unittest.TestCase):
class TestFullRunViaTasks(unittest.TestCase):
def _run(self, client, run_tag="run1", work_sub="work"):
def _run(self, client, run_tag="run1", work_sub="work", k=1, max_sessions=None):
tmp = tempfile.TemporaryDirectory()
self.addCleanup(tmp.cleanup)
root = Path(tmp.name)
@@ -221,7 +221,8 @@ class TestFullRunViaTasks(unittest.TestCase):
base_harness=dict(BASE_TREE), run_tag=run_tag,
work_dir=root / work_sub, domain="accounting",
source_files={"ledger.csv": LEDGER},
k=1, mode="parallel", library_root=root / "library")
k=k, mode="parallel", library_root=root / "library",
max_sessions=max_sessions)
return result, client, root
def test_parallel_k1_via_real_task_shape(self):
@@ -229,10 +230,11 @@ class TestFullRunViaTasks(unittest.TestCase):
result, _, root = self._run(client)
self.assertTrue(result.ok)
self.assertEqual(result.selection.chosen_label, "agg1")
self.assertTrue(result.selection.trivial) # k=1 单终版不开选择会话
self.assertEqual(result.mode_result.rollouts["agg1"].deliverable_text,
"Total: 85.15")
# 4 探针 + 3 编辑 + 1 选择 + 1 反思 = 9 个任务
self.assertEqual(client.created, 9)
# 4 探针 + 3 编辑 + 1 反思 = 8 个任务(选择 trivial,不建任务)
self.assertEqual(client.created, 8)
self.assertIsNotNone(result.library_entry)
self.assertTrue((root / "library" / "accounting" / "INDEX.md").is_file())
@@ -253,8 +255,9 @@ class TestFullRunViaTasks(unittest.TestCase):
self.assertIsNone(result.library_entry)
def test_selector_gibberish_falls_back(self):
# k=1 时选择 trivial,不走选择器;用 k=2 让选择器真正跑起来。
client = FakeTaskClient(selector_mode="gibberish")
result, _, _ = self._run(client)
result, _, _ = self._run(client, k=2)
self.assertTrue(result.ok)
self.assertTrue(result.selection.fallback)
@@ -338,6 +341,79 @@ class TestEnvelopeFullRound(unittest.TestCase):
self.assertIsNotNone(result.library_entry)
class TestSessionBudget(unittest.TestCase):
"""会话数门:0 挡全部、2 挡在中途;fmt2 与选择/反思不计入。"""
def _run(self, max_sessions):
client = FakeTaskClient()
tmp = tempfile.TemporaryDirectory()
self.addCleanup(tmp.cleanup)
root = Path(tmp.name)
result = run_improvement_via_tasks(
client, project_id="p1", task_id="ledger-1",
prompt="Sum every signed amount in ledger.csv. Return Total: with two decimals.",
base_harness=dict(BASE_TREE), run_tag="sess",
work_dir=root / "work", domain="accounting",
source_files={"ledger.csv": LEDGER},
k=1, mode="parallel", library_root=root / "library",
max_sessions=max_sessions)
return result, client
def test_zero_blocks_everything(self):
result, client = self._run(0)
self.assertEqual(client.created, 0)
self.assertEqual(set(result.mode_result.fell_back), {"inv1", "dom1", "agg1"})
def test_mid_cap_stops_after_two_sessions(self):
# base 探针(1)+ inv1 编辑(2),之后 dom1/agg1 与探针全被门挡住。
result, client = self._run(2)
self.assertEqual(client.created, 2)
self.assertIn("dom1", result.mode_result.fell_back)
self.assertIn("agg1", result.mode_result.fell_back)
self.assertNotIn("inv1", result.mode_result.fell_back)
class TestValidateRunTag(unittest.TestCase):
def test_valid(self):
for tag in ("h2-20261029-real1", "a", "a.b_c-1", "run_1"):
validate_run_tag(tag)
def test_invalid(self):
for tag in ("-bad", "has/slash", "has space", "s1-foo", "s12"):
with self.assertRaises(ValueError, msg=tag):
validate_run_tag(tag)
validate_run_tag("") # 空串表示稍后生成,不校验
class TestPreflight(unittest.TestCase):
def _dirs(self, agents="Use scripts/summarize.py.\n"):
tmp = tempfile.TemporaryDirectory()
self.addCleanup(tmp.cleanup)
root = Path(tmp.name)
(root / "base").mkdir()
if agents is not None:
(root / "base" / "AGENTS.md").write_text(agents)
(root / "source").mkdir()
(root / "source" / "ledger.csv").write_bytes(LEDGER)
return root
def test_ok(self):
root = self._dirs()
errors, _ = preflight_checks(
base_harness_dir=root / "base", source_dir=root / "source",
prompt="do it", run_tag="h2-ok")
self.assertEqual(errors, [])
def test_errors(self):
root = self._dirs(agents="")
errors, _ = preflight_checks(
base_harness_dir=root / "base", source_dir=root / "missing",
prompt=" ", run_tag="s1-bad")
self.assertEqual(len(errors), 4) # AGENTS 为空、source 缺失、提示为空、tag 非法
errors, _ = preflight_checks(base_harness_dir=root / "nope", prompt="x")
self.assertTrue(any("不存在" in e for e in errors))
class TestHttpClient(unittest.TestCase):
def test_client_against_local_server(self):
seen = {"keys": [], "uploads": [], "tasks": []}