host
159d7112ae
ci / test (push) Failing after 35m34s
Goal tasks pushed back by the audit loop lost the packet preamble on the next round: claim-time _apply_audit_loop_packet pasted it into the in-memory prompt/goal, but the next round collided with the existing host task name, fell back to breakpoint goal-resume (no --goal-text), and the host reused the stored goal, discarding the preamble. Fix (minimal, direction (c) trimmed): host goal-resume gains --resume-note, accepted only by the breakpoint branch and prepended to the first turn prompt of this resume only -- never written back to the stored goal, never touching rounds or verification. The worker stashes the same preamble as the memory-only _audit_loop_resume_note at claim time and forwards it on breakpoint resume (Local/Docker goal_resume_task and the popen goal-resume command, each guarded to ignore the note when goal_text is set for a new-goal resume). Accounting untouched: audit 3-round cap, goal_max_rounds/quota, and DB goal columns are unchanged; non-goal pushback is unchanged. Tests: cloud/tests/test_audit_n12.py (9 tests) -- host first-turn-only note + stored-goal/rounds, worker exists-fallback end-to-end with --resume-note, non-goal regression, goal-column accounting, and Local/Docker/popen command shapes. Full cloud suite: 1760 OK (skipped=11), baseline 1751 + 9 new; tests/test_host.py: 62 OK.
356 lines
15 KiB
Python
356 lines
15 KiB
Python
"""N-12：goal 任务审计推回包前言在断点续跑时丢失的修复测试。
|
||
|
||
缺口链路（批 5 定性）：认领时 _apply_audit_loop_packet 把包
|
||
前言贴进内存 prompt 与 goal，但推回后的 goal 任务下一轮无
|
||
_resume、无断点标记，先走 goal_task 撞宿主任务名已存在，再
|
||
经已存在回落走断点续跑（goal-resume 不传 --goal-text，宿主
|
||
沿用存储的原 goal），内存 goal 里的前言被覆盖、前言丢失。
|
||
|
||
修复（c 精简版）：新增 host 侧 --resume-note 参数，只由断
|
||
点分支受理，只贴进本次续跑第一轮的 prompt，绝不写回存储
|
||
的 goal、不改轮次与验收；worker 认领贴前言时另存内存键
|
||
_audit_loop_resume_note，断点续跑时作 --resume-note 透传
|
||
（Local/Docker 的 goal_resume_task 与 popen 的 goal-resume
|
||
命令构造各有配对守卫：goal_text 已设时忽略备注）。
|
||
|
||
只用注入的假 runner / 假 server，不跑真实模型回合、不联
|
||
网、不登录任何服务。断言口径：包前言渲染的是 findings
|
||
（id/证据/复现/验收）与固定收尾句，criteria 是包里的门映
|
||
射（由接线会计用，不渲染进前言文本），所以“前言到达”
|
||
断言为前言文本含 finding id 与证据/验收句；criteria 仍在
|
||
包 JSON 里可读、接线会计不变。
|
||
"""
|
||
|
||
import json
|
||
import os
|
||
import sqlite3
|
||
import sys
|
||
import tempfile
|
||
import time
|
||
import unittest
|
||
from pathlib import Path
|
||
from types import SimpleNamespace
|
||
from unittest import mock
|
||
|
||
try:
|
||
from cloud.sandbox import DockerExecutor, LocalExecutor
|
||
from cloud.server import init_db
|
||
from cloud.worker import Worker
|
||
except ImportError: # discover -s cloud/tests 时把项目根目录补进 sys.path
|
||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||
from cloud.sandbox import DockerExecutor, LocalExecutor
|
||
from cloud.server import init_db
|
||
from cloud.worker import Worker
|
||
|
||
|
||
def _host_mod():
|
||
"""host.py 只在测试执行时才导入，不在模块层导入：host.py
|
||
有 `import queue`（标准库），模块层导入会先于
|
||
test_queue_sandbox 的模块导入把标准库 queue 写进
|
||
sys.modules，导致后者 `from queue import TaskQueue`
|
||
（本意是 cloud/queue.py）拿到标准库而导入失败、47 项
|
||
用例缺席。全部模块发现完成后再导入 host 就不受影响
|
||
（test_queue_sandbox 的名字在发现时已绑定 cloud/queue）。"""
|
||
try:
|
||
import host as host_mod
|
||
except ImportError:
|
||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||
import host as host_mod
|
||
return host_mod
|
||
|
||
|
||
def _packet(task_id="TG", round_no=1):
|
||
return json.dumps({
|
||
"task_id": task_id, "round": round_no,
|
||
"findings": [{"id": "execution_failed", "severity": "high",
|
||
"evidence": "boom detail", "reproduction": "re-run",
|
||
"acceptance": "task completes",
|
||
"kind": "deterministic", "gate": "execution"}],
|
||
"criteria": {"execution_failed": "execution"}})
|
||
|
||
|
||
def _insert_task(db_path, task_id, prompt="do work", status="queued",
|
||
project_id="p1", audit_loop=0, packet=None,
|
||
goal_mode=0, goal=None, verify_command=None,
|
||
goal_max_rounds=None):
|
||
now = time.time()
|
||
conn = sqlite3.connect(db_path)
|
||
try:
|
||
conn.execute(
|
||
"INSERT INTO tasks(id, project_id, prompt, status,"
|
||
" idempotency_key, depends_on, priority, created_by,"
|
||
" created_at, updated_at, last_heartbeat, audit_loop,"
|
||
" audit_loop_packet, goal_mode, goal, verify_command,"
|
||
" goal_max_rounds)"
|
||
" VALUES(?, ?, ?, ?, NULL, '[]', 0, 'tester', ?, ?, NULL,"
|
||
" ?, ?, ?, ?, ?, ?)",
|
||
(task_id, project_id, prompt, status, now, now,
|
||
audit_loop, packet, goal_mode, goal, verify_command,
|
||
goal_max_rounds),
|
||
)
|
||
conn.commit()
|
||
finally:
|
||
conn.close()
|
||
|
||
|
||
def _row(db_path, task_id):
|
||
conn = sqlite3.connect(db_path)
|
||
try:
|
||
return conn.execute(
|
||
"SELECT status, prompt, goal, verify_command,"
|
||
" goal_max_rounds, audit_loop_packet"
|
||
" FROM tasks WHERE id = ?", (task_id,)).fetchone()
|
||
finally:
|
||
conn.close()
|
||
|
||
|
||
class _FakeServer:
|
||
"""只记 prompt、不跑模型：run_turn 返回 completed。"""
|
||
|
||
def __init__(self):
|
||
self.prompts = []
|
||
|
||
def run_turn(self, thread_id, prompt, **kwargs):
|
||
self.prompts.append(prompt)
|
||
return {"status": "completed", "text": "did work", "usage": {}}
|
||
|
||
|
||
def _host_state(tmp, task):
|
||
state_path = os.path.join(tmp, "host-state.json")
|
||
state = {"tasks": {task["name"]: task}, "automations": [],
|
||
"usage": {}}
|
||
return state, state_path
|
||
|
||
|
||
class HostResumeNoteTests(unittest.TestCase):
|
||
"""host 侧：--resume-note 只贴首轮、不写回存储 goal、不改轮次。"""
|
||
|
||
def test_breakpoint_note_first_turn_only_and_rounds_continue(self):
|
||
host_mod = _host_mod()
|
||
srv = _FakeServer()
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
task = {"name": "g", "thread_id": "x", "cwd": tmp,
|
||
"sandbox": None, "max_retries": 0, "retry_count": 0,
|
||
"turns": [], "status": "goal-running",
|
||
"goal_rounds_done": 1}
|
||
state, state_path = _host_state(tmp, task)
|
||
note = ("[审计推回包（第 1 轮）]\n"
|
||
"- [execution_failed] 证据：boom detail")
|
||
host_mod.run_goal_loop(
|
||
srv, state, state_path, task, "orig goal", "false",
|
||
3, "m", "low", None, start_round=2,
|
||
last_verify="", last_turn_status=None,
|
||
last_turn_text="", resume_note=note)
|
||
self.assertEqual(len(srv.prompts), 2)
|
||
self.assertTrue(srv.prompts[0].startswith(note))
|
||
self.assertNotIn("execution_failed", srv.prompts[1])
|
||
# 存储的 goal 不变（备注只是本次续跑首轮的展示前缀），
|
||
# 轮次从断点续接：1（已做）+ 2 轮 = 3，不重置、不跳号。
|
||
self.assertEqual(task["goal"], "orig goal")
|
||
self.assertEqual(task["goal_rounds_done"], 3)
|
||
self.assertEqual(task["status"], "goal-unmet")
|
||
|
||
def test_met_path_note_first_turn_only_and_goal_unchanged(self):
|
||
host_mod = _host_mod()
|
||
srv = _FakeServer()
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
task = {"name": "g", "thread_id": "x", "cwd": tmp,
|
||
"sandbox": None, "max_retries": 0, "retry_count": 0,
|
||
"turns": []}
|
||
state, state_path = _host_state(tmp, task)
|
||
note = "[审计推回包（第 1 轮）]\n- [execution_failed]"
|
||
result = host_mod.run_goal_loop(
|
||
srv, state, state_path, task, "orig goal", "true",
|
||
3, "m", "low", None, resume_note=note)
|
||
self.assertTrue(result["ok"])
|
||
self.assertEqual(len(srv.prompts), 1)
|
||
self.assertTrue(srv.prompts[0].startswith(note))
|
||
self.assertIn("orig goal", srv.prompts[0])
|
||
self.assertEqual(task["goal"], "orig goal")
|
||
|
||
def test_no_note_behavior_unchanged(self):
|
||
host_mod = _host_mod()
|
||
srv = _FakeServer()
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
task = {"name": "g", "thread_id": "x", "cwd": tmp,
|
||
"sandbox": None, "max_retries": 0, "retry_count": 0,
|
||
"turns": []}
|
||
state, state_path = _host_state(tmp, task)
|
||
host_mod.run_goal_loop(
|
||
srv, state, state_path, task, "orig goal", "true",
|
||
3, "m", "low", None)
|
||
self.assertEqual(srv.prompts, ["orig goal"])
|
||
self.assertEqual(task["goal"], "orig goal")
|
||
|
||
|
||
class WorkerResumeNoteTests(unittest.TestCase):
|
||
"""worker 侧：推回 goal 任务经已存在回落走断点续跑时带备注。"""
|
||
|
||
def setUp(self):
|
||
self.tmp = tempfile.TemporaryDirectory()
|
||
self.db_path = os.path.join(self.tmp.name, "control.db")
|
||
init_db(self.db_path)
|
||
self.env = {"XCM_API_KEY": "test-placeholder"}
|
||
self.seen = []
|
||
|
||
def tearDown(self):
|
||
self.tmp.cleanup()
|
||
|
||
def _worker(self, runner):
|
||
return Worker(self.db_path, "/fake/host.py", runner=runner,
|
||
env=self.env, max_retries=0,
|
||
retry_backoff_seconds=0)
|
||
|
||
def _fake_runner(self, cmd):
|
||
self.seen.append(list(cmd))
|
||
if "goal-resume" in cmd:
|
||
return SimpleNamespace(
|
||
returncode=0,
|
||
stdout="round 1: turn=completed verify_rc=-\n"
|
||
"goal met after rounds\n",
|
||
stderr="")
|
||
return SimpleNamespace(returncode=0,
|
||
stdout="task 已存在", stderr="")
|
||
|
||
def test_pushback_goal_fallback_resume_carries_note(self):
|
||
_insert_task(self.db_path, "TG", prompt="do goal work",
|
||
audit_loop=1, packet=_packet(), goal_mode=1,
|
||
goal="orig goal", verify_command="false",
|
||
goal_max_rounds=3)
|
||
worker = self._worker(self._fake_runner)
|
||
result = worker.run_one()
|
||
self.assertEqual(result["status"], "completed", result)
|
||
# 第一程 goal（撞已存在），第二程断点续跑（带备注）。
|
||
self.assertEqual(len(self.seen), 2)
|
||
self.assertIn("goal", self.seen[0])
|
||
self.assertNotIn("goal-resume", self.seen[0])
|
||
resume_cmd = self.seen[1]
|
||
self.assertIn("goal-resume", resume_cmd)
|
||
self.assertNotIn("--goal-text", resume_cmd)
|
||
self.assertIn("--resume-note", resume_cmd)
|
||
note = resume_cmd[resume_cmd.index("--resume-note") + 1]
|
||
self.assertIn("execution_failed", note)
|
||
self.assertIn("boom detail", note)
|
||
self.assertIn("task completes", note)
|
||
# 库内原 goal / prompt / 验收 / 轮次上限一律不动（备注只
|
||
# 走本次续跑输入，不写回库）。
|
||
row = _row(self.db_path, "TG")
|
||
self.assertEqual(row[0], "completed")
|
||
self.assertEqual(row[1], "do goal work")
|
||
self.assertEqual(row[2], "orig goal")
|
||
self.assertEqual(row[3], "false")
|
||
self.assertEqual(row[4], 3)
|
||
|
||
def test_apply_packet_keeps_goal_columns_and_stashes_note(self):
|
||
worker = self._worker(self._fake_runner)
|
||
task = {"id": "TG", "project_id": "p1", "prompt": "do goal work",
|
||
"goal": "orig goal", "goal_mode": 1,
|
||
"verify_command": "false", "goal_max_rounds": 3,
|
||
"audit_loop_packet": _packet()}
|
||
out = worker._apply_audit_loop_packet(task)
|
||
self.assertTrue(out["prompt"].startswith("[审计推回包"))
|
||
self.assertIn("execution_failed", out["goal"])
|
||
self.assertEqual(out["verify_command"], "false")
|
||
self.assertEqual(out["goal_max_rounds"], 3)
|
||
self.assertIn("execution_failed",
|
||
out["_audit_loop_resume_note"])
|
||
again = worker._apply_audit_loop_packet(out)
|
||
self.assertEqual(again["prompt"].count("execution_failed"), 1)
|
||
|
||
def test_non_goal_pushback_unchanged(self):
|
||
_insert_task(self.db_path, "TP", prompt="do work",
|
||
audit_loop=1, packet=_packet(task_id="TP"))
|
||
worker = self._worker(self._fake_runner)
|
||
task = worker.get_task("TP")
|
||
out = worker._apply_audit_loop_packet(task)
|
||
self.assertTrue(out["prompt"].startswith("[审计推回包"))
|
||
self.assertIn("execution_failed", out["prompt"])
|
||
# 库内原 prompt 不动（只改内存副本）。
|
||
self.assertEqual(_row(self.db_path, "TP")[1], "do work")
|
||
|
||
|
||
class CommandShapeTests(unittest.TestCase):
|
||
"""命令形状：有备注时含 --resume-note，无备注时无，
|
||
goal_text 已设（开新目标）时忽略备注。"""
|
||
|
||
def test_local_goal_resume_note_shape(self):
|
||
calls = []
|
||
|
||
def fake(cmd):
|
||
calls.append(list(cmd))
|
||
return SimpleNamespace(
|
||
returncode=0,
|
||
stdout="goal met after rounds\n",
|
||
stderr="")
|
||
|
||
task = {"id": "T1", "project_id": "p1", "prompt": "orig goal",
|
||
"verify_command": "true", "goal_max_rounds": 3}
|
||
local = LocalExecutor("/fake/host.py", runner=fake)
|
||
local.goal_resume_task(task, None, resume_note="NOTE")
|
||
self.assertIn("--resume-note", calls[-1])
|
||
self.assertIn("NOTE", calls[-1])
|
||
self.assertNotIn("--goal-text", calls[-1])
|
||
local.goal_resume_task(task, None)
|
||
self.assertNotIn("--resume-note", calls[-1])
|
||
local.goal_resume_task(task, "new goal", resume_note="NOTE")
|
||
self.assertIn("--goal-text", calls[-1])
|
||
self.assertNotIn("--resume-note", calls[-1])
|
||
|
||
def test_docker_goal_resume_note_shape(self):
|
||
calls = []
|
||
|
||
def dfake(cmd, timeout=None):
|
||
calls.append(list(cmd))
|
||
return SimpleNamespace(
|
||
returncode=0,
|
||
stdout="goal met after rounds\n",
|
||
stderr="")
|
||
|
||
task = {"id": "T1", "project_id": "p1", "prompt": "orig goal",
|
||
"verify_command": "true", "goal_max_rounds": 3}
|
||
docker = DockerExecutor(
|
||
runner=dfake, env={"XCM_API_KEY": "test-placeholder"})
|
||
with mock.patch("cloud.sandbox.shutil.which",
|
||
return_value="/usr/bin/docker"):
|
||
docker.goal_resume_task(task, None, resume_note="NOTE")
|
||
self.assertIn("--resume-note", calls[-1])
|
||
self.assertIn("NOTE", calls[-1])
|
||
with mock.patch("cloud.sandbox.shutil.which",
|
||
return_value="/usr/bin/docker"):
|
||
docker.goal_resume_task(task, None)
|
||
self.assertNotIn("--resume-note", calls[-1])
|
||
with mock.patch("cloud.sandbox.shutil.which",
|
||
return_value="/usr/bin/docker"):
|
||
docker.goal_resume_task(task, "new goal",
|
||
resume_note="NOTE")
|
||
self.assertIn("--goal-text", calls[-1])
|
||
self.assertNotIn("--resume-note", calls[-1])
|
||
|
||
def test_popen_goal_resume_breakpoint_note_shape(self):
|
||
local = LocalExecutor("/fake/host.py", runner=lambda cmd: None)
|
||
with mock.patch.object(
|
||
__import__("cloud.sandbox", fromlist=["subprocess"])
|
||
.subprocess, "Popen") as popen:
|
||
popen.return_value = SimpleNamespace(pid=9999)
|
||
local.popen_task(
|
||
{"id": "T1", "prompt": "resume msg",
|
||
"_resume_kind": "breakpoint",
|
||
"_audit_loop_resume_note": "NOTE"},
|
||
kind="goal-resume")
|
||
cmd = popen.call_args[0][0]
|
||
self.assertIn("goal-resume", cmd)
|
||
self.assertNotIn("--goal-text", cmd)
|
||
self.assertIn("--resume-note", cmd)
|
||
self.assertIn("NOTE", cmd)
|
||
# 无备注的断点续跑：命令形状与旧版一致。
|
||
local.popen_task(
|
||
{"id": "T1", "prompt": "resume msg",
|
||
"_resume_kind": "breakpoint"},
|
||
kind="goal-resume")
|
||
cmd = popen.call_args[0][0]
|
||
self.assertNotIn("--resume-note", cmd)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
unittest.main()
|