host
ae3aff8602
1. host goal 循环连续失败护栏：3 轮连续失败即中断出口， 任务记 interrupted、可断点续跑，不再无限空转 （2026-10-29 实弹：断点后 274 轮全失败）。 2. worker 退出杀残留路径补审计 worker_shutdown_interrupted， detail 为任务 id，与 sweep_interrupted 同口径。 3. 配额中断自动恢复：每轮现查配额，仍超限/查不到时留 在 interrupted 不重排队，下轮再查；现查为 None 时 （新月份/额度上调/无价上限重置）才 CAS 回 queued。 单测：配额仍超限不动、配额恢复续跑、连续 3 轮失败停转。
1263 lines
58 KiB
Python
1263 lines
58 KiB
Python
import json
|
||
import os
|
||
import shutil
|
||
import subprocess
|
||
import sys
|
||
import tempfile
|
||
import unittest
|
||
from pathlib import Path
|
||
|
||
|
||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||
sys.path.insert(0, str(REPO_ROOT))
|
||
|
||
import host
|
||
|
||
|
||
class EffectiveCfgTest(unittest.TestCase):
|
||
# G9：--ignore-host-budget 只清 max_daily_turns，其它配置不动。
|
||
def test_ignore_host_budget_clears_only_budget(self):
|
||
cfg = {"max_daily_turns": 10, "notify_command": "echo hi",
|
||
"model": "gpt-6.1-sol"}
|
||
out = host._effective_cfg(cfg, True)
|
||
self.assertIsNone(out["max_daily_turns"])
|
||
self.assertEqual(out["notify_command"], "echo hi")
|
||
self.assertEqual(out["model"], "gpt-6.1-sol")
|
||
# 不改原 cfg。
|
||
self.assertEqual(cfg["max_daily_turns"], 10)
|
||
|
||
def test_no_ignore_returns_cfg_unchanged(self):
|
||
cfg = {"max_daily_turns": 10}
|
||
self.assertEqual(host._effective_cfg(cfg, False), cfg)
|
||
|
||
|
||
class HostPureFunctionsTest(unittest.TestCase):
|
||
def test_save_state_merges_tasks_and_preserves_disk_tasks(self):
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
state_path = os.path.join(tmp, "host-state.json")
|
||
with open(state_path, "w", encoding="utf-8") as f:
|
||
json.dump({"tasks": {"disk-task": {"status": "running"}},
|
||
"automations": []}, f)
|
||
|
||
state = {"tasks": {"memory-task": {"status": "completed"}},
|
||
"automations": []}
|
||
host.save_state(state_path, state)
|
||
|
||
self.assertEqual(state["tasks"]["disk-task"], {"status": "running"})
|
||
self.assertEqual(state["tasks"]["memory-task"], {"status": "completed"})
|
||
loaded = host.load_state(state_path)
|
||
self.assertEqual(loaded, state)
|
||
|
||
def test_task_diff_includes_untracked_text_contents(self):
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
subprocess.run(["git", "init", "-q"], cwd=tmp, check=True)
|
||
Path(tmp, "tracked.txt").write_text("tracked\n", encoding="utf-8")
|
||
subprocess.run(["git", "add", "tracked.txt"], cwd=tmp, check=True)
|
||
subprocess.run(["git", "-c", "user.name=Test", "-c",
|
||
"user.email=test@example.com", "commit", "-qm",
|
||
"initial"], cwd=tmp, check=True)
|
||
Path(tmp, "new.txt").write_text("new file contents\n", encoding="utf-8")
|
||
|
||
result = host.task_diff(tmp)
|
||
|
||
self.assertIn("new.txt", result["untracked"].splitlines())
|
||
self.assertEqual(result["untracked_contents"]["new.txt"],
|
||
"new file contents\n")
|
||
|
||
def test_load_config_file_returns_empty_dict_for_missing_file(self):
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
missing = os.path.join(tmp, "missing.json")
|
||
self.assertEqual(host._load_config_file(missing), {})
|
||
|
||
def test_server_command_excludes_secrets_from_agent_shell(self):
|
||
cmd = host._server_command("/usr/bin/codex")
|
||
self.assertEqual(cmd[0], "/usr/bin/codex")
|
||
self.assertEqual(cmd[-1], "app-server")
|
||
self.assertIn("-c", cmd)
|
||
flag = cmd[cmd.index("-c") + 1]
|
||
self.assertTrue(
|
||
flag.startswith("shell_environment_policy.exclude="), flag)
|
||
excluded = json.loads(
|
||
flag.split("=", 1)[1])
|
||
for name in ("XCM_API_KEY", "CLOUD_DB_PASSWORD",
|
||
"GITHUB_OAUTH_CLIENT_SECRET", "GITHUB_WEBHOOK_SECRET",
|
||
"*_KEY", "*_SECRET", "*_PASSWORD"):
|
||
self.assertIn(name, excluded)
|
||
|
||
|
||
def _init_git_repo(tmp):
|
||
subprocess.run(["git", "init", "-q"], cwd=tmp, check=True)
|
||
subprocess.run(["git", "config", "user.name", "Test"], cwd=tmp, check=True)
|
||
subprocess.run(["git", "config", "user.email", "test@example.com"],
|
||
cwd=tmp, check=True)
|
||
|
||
|
||
class TopoSortTest(unittest.TestCase):
|
||
def test_priority_and_dependency_order(self):
|
||
jobs = [
|
||
{"name": "low", "depends_on": [], "priority": 0},
|
||
{"name": "high", "depends_on": [], "priority": 5},
|
||
{"name": "child", "depends_on": ["low", "high"], "priority": 0},
|
||
]
|
||
order = [j["name"] for j in host.topo_sort_jobs(jobs)]
|
||
self.assertEqual(order[0], "high")
|
||
self.assertEqual(order[-1], "child")
|
||
|
||
def test_cycle_and_missing_dependency_raise(self):
|
||
with self.assertRaises(ValueError):
|
||
host.topo_sort_jobs([{"name": "a", "depends_on": ["b"]}])
|
||
with self.assertRaises(ValueError):
|
||
host.topo_sort_jobs([
|
||
{"name": "a", "depends_on": ["b"]},
|
||
{"name": "b", "depends_on": ["a"]},
|
||
])
|
||
|
||
|
||
class MergeVerifyTest(unittest.TestCase):
|
||
def _make_repo(self, tmp, break_test=False):
|
||
_init_git_repo(tmp)
|
||
Path(tmp, "mod.py").write_text("def f():\n return 1\n",
|
||
encoding="utf-8")
|
||
tests_dir = Path(tmp, "tests")
|
||
tests_dir.mkdir()
|
||
body = ("import unittest\nfrom mod import f\n\n"
|
||
"class T(unittest.TestCase):\n"
|
||
" def test_f(self):\n"
|
||
" self.assertEqual(f(), 1)\n")
|
||
(tests_dir / "test_mod.py").write_text(body, encoding="utf-8")
|
||
subprocess.run(["git", "add", "-A"], cwd=tmp, check=True)
|
||
subprocess.run(["git", "-c", "user.name=Test", "-c", "user.email=test@example.com", "commit", "-qm", "initial"], cwd=tmp,
|
||
check=True)
|
||
# worktree 手动建：模拟 host 的 worktree/分支
|
||
subprocess.run(["git", "worktree", "add", os.path.join(tmp, "wt"),
|
||
"-b", "host/t1", "HEAD"], cwd=tmp, check=True)
|
||
if break_test:
|
||
Path(tmp, "wt", "mod.py").write_text(
|
||
"def f():\n return 2\n", encoding="utf-8")
|
||
else:
|
||
Path(tmp, "wt", "mod.py").write_text(
|
||
"def f():\n return 1 # comment\n", encoding="utf-8")
|
||
return os.path.join(tmp, "wt")
|
||
|
||
def test_merge_verify_pass_cleans_worktree(self):
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
wt = self._make_repo(tmp)
|
||
ok, msg = host.merge_task(tmp, wt, "host/t1", "t1",
|
||
verify_command=None)
|
||
self.assertTrue(ok, msg)
|
||
self.assertFalse(os.path.isdir(wt))
|
||
out = subprocess.run(["git", "branch", "--list", "host/t1"],
|
||
cwd=tmp, capture_output=True, text=True)
|
||
self.assertEqual(out.stdout.strip(), "")
|
||
|
||
def test_merge_verify_fail_rolls_back_and_keeps_worktree(self):
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
wt = self._make_repo(tmp, break_test=True)
|
||
pre = subprocess.run(["git", "rev-parse", "HEAD"], cwd=tmp,
|
||
capture_output=True, text=True).stdout.strip()
|
||
ok, msg = host.merge_task(tmp, wt, "host/t1", "t1",
|
||
verify_command=None)
|
||
self.assertFalse(ok)
|
||
self.assertIn("回滚", msg)
|
||
now = subprocess.run(["git", "rev-parse", "HEAD"], cwd=tmp,
|
||
capture_output=True, text=True).stdout.strip()
|
||
self.assertEqual(now, pre)
|
||
self.assertTrue(os.path.isdir(wt))
|
||
self.assertEqual(Path(tmp, "mod.py").read_text(),
|
||
"def f():\n return 1\n")
|
||
|
||
def test_merge_duplicate_add_auto_resolved_with_ours(self):
|
||
# 依赖任务场景：两个分支都在同一位置加了同一段 add2，
|
||
# 先合并第一个，再合并第二个时应自动解（-X ours），
|
||
# 且第二个独有的 mul2 仍在主仓里。
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
_init_git_repo(tmp)
|
||
Path(tmp, "mod.py").write_text("def f():\n return 1\n",
|
||
encoding="utf-8")
|
||
subprocess.run(["git", "add", "-A"], cwd=tmp, check=True)
|
||
subprocess.run(["git", "-c", "user.name=Test", "-c", "user.email=test@example.com", "commit", "-qm", "initial"], cwd=tmp,
|
||
check=True)
|
||
# 分支1：加 add2
|
||
subprocess.run(["git", "worktree", "add",
|
||
os.path.join(tmp, "wt1"), "-b", "host/t1",
|
||
"HEAD"], cwd=tmp, check=True)
|
||
Path(tmp, "wt1", "mod.py").write_text(
|
||
"def f():\n return 1\n\ndef add2(a):\n return a + 2\n",
|
||
encoding="utf-8")
|
||
ok, _ = host.merge_task(tmp, os.path.join(tmp, "wt1"),
|
||
"host/t1", "t1", verify_command="")
|
||
self.assertTrue(ok)
|
||
# 分支2：从初始 HEAD 建，但拷了分支1的改动 + 自己的 mul2
|
||
subprocess.run(["git", "worktree", "add",
|
||
os.path.join(tmp, "wt2"), "-b", "host/t2",
|
||
"HEAD"], cwd=tmp, check=True)
|
||
Path(tmp, "wt2", "mod.py").write_text(
|
||
"def f():\n return 1\n\ndef add2(a):\n return a + 2\n\n"
|
||
"def mul2(a):\n return a * 2\n", encoding="utf-8")
|
||
ok, msg = host.merge_task(tmp, os.path.join(tmp, "wt2"),
|
||
"host/t2", "t2", verify_command="")
|
||
self.assertTrue(ok, msg)
|
||
content = Path(tmp, "mod.py").read_text()
|
||
self.assertIn("def add2", content)
|
||
self.assertIn("def mul2", content)
|
||
# add2 只出现一次，不是重复两份
|
||
self.assertEqual(content.count("def add2"), 1)
|
||
|
||
def test_merge_does_not_commit_pycache(self):
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
wt = self._make_repo(tmp)
|
||
# 模拟 agent 跑测试生成的缓存
|
||
pyc_dir = Path(wt, "__pycache__")
|
||
pyc_dir.mkdir(exist_ok=True)
|
||
(pyc_dir / "mod.cpython-312.pyc").write_bytes(b"fake-pyc")
|
||
ok, msg = host.merge_task(tmp, wt, "host/t1", "t1",
|
||
verify_command="")
|
||
self.assertTrue(ok, msg)
|
||
tracked = subprocess.run(["git", "ls-files"], cwd=tmp,
|
||
capture_output=True, text=True).stdout
|
||
self.assertNotIn("pyc", tracked)
|
||
self.assertNotIn("__pycache__", tracked)
|
||
|
||
|
||
class ReviewCommentTest(unittest.TestCase):
|
||
def test_comment_and_fix_prompt(self):
|
||
task = {"name": "t", "review_comments": []}
|
||
host.add_review_comment(task, "mod.py", 12, "加 docstring")
|
||
host.add_review_comment(task, "mod.py", 20, "处理空输入")
|
||
self.assertEqual(len(host.open_review_comments(task)), 2)
|
||
prompt = host.build_fix_prompt(task)
|
||
self.assertIn("mod.py:12: 加 docstring", prompt)
|
||
self.assertIn("mod.py:20: 处理空输入", prompt)
|
||
|
||
|
||
class RetryAndBudgetTest(unittest.TestCase):
|
||
class FakeServer:
|
||
def __init__(self, results):
|
||
self.results = list(results)
|
||
self.calls = 0
|
||
|
||
def run_turn(self, thread_id, prompt, model=None, effort=None,
|
||
sandbox=None):
|
||
self.calls += 1
|
||
return self.results.pop(0)
|
||
|
||
def test_retry_then_success(self):
|
||
task = {"name": "t", "thread_id": "x", "sandbox": None,
|
||
"max_retries": 2, "retry_count": 0, "turns": []}
|
||
state = {"tasks": {"t": task}, "automations": []}
|
||
srv = self.FakeServer([
|
||
{"status": "failed", "text": ""},
|
||
{"status": "completed", "text": "ok"},
|
||
])
|
||
result = host.run_task_turn_with_retry(srv, state, task, "do it",
|
||
"m", "low", cfg={})
|
||
self.assertEqual(result["status"], "completed")
|
||
self.assertEqual(task["retry_count"], 1)
|
||
self.assertEqual(srv.calls, 2)
|
||
|
||
def test_budget_blocks_before_turn(self):
|
||
task = {"name": "t", "thread_id": "x", "sandbox": None,
|
||
"max_retries": 0, "retry_count": 0, "turns": []}
|
||
today = __import__("time").strftime("%Y-%m-%d")
|
||
state = {"tasks": {"t": task}, "automations": [],
|
||
"usage": {"date": today, "turns": 5}}
|
||
srv = self.FakeServer([{"status": "completed", "text": "ok"}])
|
||
result = host.run_task_turn_with_retry(
|
||
srv, state, task, "do it", "m", "low",
|
||
cfg={"max_daily_turns": 5})
|
||
self.assertEqual(result["status"], "failed")
|
||
self.assertEqual(srv.calls, 0)
|
||
|
||
def test_notify_failure_writes_log_and_runs_command(self):
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
state_path = os.path.join(tmp, "host-state.json")
|
||
notify_log = os.path.join(tmp, "notify.log")
|
||
cfg = {"notify_command": f"echo {{task}} {{status}} >> {notify_log}"}
|
||
host.notify_failure(state_path, cfg, "t1",
|
||
{"status": "failed", "text": "boom"})
|
||
self.assertTrue(os.path.exists(
|
||
os.path.join(tmp, "host-run.log")))
|
||
self.assertIn("t1 failed", Path(notify_log).read_text())
|
||
|
||
def test_notify_failure_quotes_shell_metacharacters(self):
|
||
# 任务名 / 文本里的 shell 元字符必须被 quote，不能当命令执行：
|
||
# 未加 quote 时 task 名里的 `; touch pwn` 会真的建出文件。
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
state_path = os.path.join(tmp, "host-state.json")
|
||
notify_log = os.path.join(tmp, "notify.log")
|
||
pwn = os.path.join(tmp, "pwn")
|
||
pwn2 = os.path.join(tmp, "pwn2")
|
||
cfg = {"notify_command":
|
||
f"echo {{task}} {{status}} {{text}} >> {notify_log}"}
|
||
host.notify_failure(
|
||
state_path, cfg, "t1; touch " + pwn,
|
||
{"status": "failed", "text": "$(touch " + pwn2 + ")"})
|
||
self.assertFalse(os.path.exists(pwn))
|
||
self.assertFalse(os.path.exists(pwn2))
|
||
self.assertIn("t1; touch", Path(notify_log).read_text())
|
||
|
||
def test_notify_failure_bad_template_is_logged_not_swallowed(self):
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
state_path = os.path.join(tmp, "host-state.json")
|
||
notify_log = os.path.join(tmp, "notify.log")
|
||
cfg = {"notify_command": f"echo {{unknown}} >> {notify_log}"}
|
||
host.notify_failure(state_path, cfg, "t1",
|
||
{"status": "failed", "text": "boom"})
|
||
# 模板错了：不跑命令（notify.log 不应出现），但必须
|
||
# 在 host-run.log 里留下 template error，而不是静默吞掉。
|
||
self.assertFalse(os.path.exists(notify_log))
|
||
self.assertIn("notify command template error",
|
||
Path(os.path.join(tmp, "host-run.log")).read_text())
|
||
|
||
def test_notify_failure_command_failure_is_logged(self):
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
state_path = os.path.join(tmp, "host-state.json")
|
||
cfg = {"notify_command": "exit 3"}
|
||
host.notify_failure(state_path, cfg, "t1",
|
||
{"status": "failed", "text": "boom"})
|
||
self.assertIn("notify command rc=3",
|
||
Path(os.path.join(tmp, "host-run.log")).read_text())
|
||
|
||
def test_goal_budget_stop_notifies_budget_exceeded(self):
|
||
class FakeServer:
|
||
def run_turn(self, *args, **kwargs):
|
||
raise AssertionError("预算已满，不应再跑回合")
|
||
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
state_path = os.path.join(tmp, "host-state.json")
|
||
notify_log = os.path.join(tmp, "notify.log")
|
||
today = __import__("time").strftime("%Y-%m-%d")
|
||
state = {"tasks": {}, "automations": [],
|
||
"usage": {"date": today, "turns": 5}}
|
||
task = {"name": "g", "thread_id": "x", "cwd": tmp,
|
||
"sandbox": None, "max_retries": 0, "retry_count": 0,
|
||
"turns": []}
|
||
state["tasks"]["g"] = task
|
||
cfg = {"max_daily_turns": 5,
|
||
"notify_command":
|
||
f"echo {{task}} {{status}} >> {notify_log}"}
|
||
result = host.run_goal_loop(FakeServer(), state, state_path,
|
||
task, "make it", "true", 3, "m",
|
||
"low", None, cfg=cfg, timeout=600)
|
||
self.assertFalse(result["ok"])
|
||
self.assertTrue(result["budget_stopped"])
|
||
self.assertEqual(task["status"], "interrupted")
|
||
self.assertIn("budget-exceeded",
|
||
Path(notify_log).read_text())
|
||
|
||
def test_goal_turn_exception_is_interrupted_not_failed(self):
|
||
# 2026-10-28 长 goal 实弹：回合抛错必须记 interrupted +
|
||
# 打印标记行（云端据此可断点续跑），绝不能向上传播致
|
||
# 进程非 0 退出、无结论行（旧语义云端判 terminal failed）。
|
||
import contextlib
|
||
import io
|
||
|
||
class FakeServer:
|
||
def run_turn(self, *args, **kwargs):
|
||
raise RuntimeError("relay boom")
|
||
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
state_path = os.path.join(tmp, "host-state.json")
|
||
state = {"tasks": {}, "automations": [], "usage": {}}
|
||
task = {"name": "g", "thread_id": "x", "cwd": tmp,
|
||
"sandbox": None, "max_retries": 0, "retry_count": 0,
|
||
"turns": []}
|
||
state["tasks"]["g"] = task
|
||
buf = io.StringIO()
|
||
with contextlib.redirect_stdout(buf):
|
||
result = host.run_goal_loop(
|
||
FakeServer(), state, state_path,
|
||
task, "make it", "true", 3, "m",
|
||
"low", None, timeout=600)
|
||
self.assertTrue(result["interrupted"])
|
||
self.assertFalse(result["ok"])
|
||
self.assertEqual(task["status"], "interrupted")
|
||
self.assertEqual(task["goal_rounds_done"], 0)
|
||
self.assertIn("relay boom",
|
||
task["goal_interrupted_reason"])
|
||
self.assertIn("goal interrupted after rounds",
|
||
buf.getvalue())
|
||
saved = json.loads(Path(state_path).read_text())
|
||
self.assertEqual(saved["tasks"]["g"]["status"],
|
||
"interrupted")
|
||
|
||
|
||
class TurnErrorFromHistoryTest(unittest.TestCase):
|
||
def _server_with_history(self, tmp, error_json):
|
||
import sqlite3
|
||
db = os.path.join(tmp, "thread_history_1.sqlite")
|
||
conn = sqlite3.connect(db)
|
||
conn.execute(
|
||
"CREATE TABLE thread_turns(thread_id TEXT, turn_id TEXT,"
|
||
" error_json TEXT)")
|
||
if error_json is not None:
|
||
conn.execute(
|
||
"INSERT INTO thread_turns VALUES('th1', 'tu1', ?)",
|
||
(error_json,))
|
||
conn.commit()
|
||
conn.close()
|
||
srv = object.__new__(host.Server)
|
||
srv.codex_home = tmp
|
||
return srv
|
||
|
||
def test_failed_turn_recovers_404_from_history(self):
|
||
# 事件里 status=failed 但 error 为空时，从 thread_history
|
||
# 库里按 thread_id + turn_id 捞回 404 原因（主机实测回归：
|
||
# 坏模型回合的 404 只在库里，事件里没有，导致 worker
|
||
# 永远看不到可 fallback 信号）。
|
||
import json as _json
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
err = _json.dumps({
|
||
"message": "unexpected status 404 Not Found: Model "
|
||
"\"gpt-9.9-nonexistent\" is not available",
|
||
"codexErrorInfo": {"httpConnectionFailed":
|
||
{"httpStatusCode": 404}}})
|
||
srv = self._server_with_history(tmp, err)
|
||
got = srv._turn_error_from_history("th1", "tu1")
|
||
self.assertIn("404", got)
|
||
self.assertIn("gpt-9.9-nonexistent", got)
|
||
|
||
def test_missing_history_returns_none_without_crashing(self):
|
||
# 库不存在、行不存在、JSON 损坏都只返回 None，不抛错。
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
srv = self._server_with_history(tmp, None)
|
||
self.assertIsNone(srv._turn_error_from_history("th1", "tu1"))
|
||
self.assertIsNone(srv._turn_error_from_history("th1", None))
|
||
self.assertIsNone(srv._turn_error_from_history(None, "tu1"))
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
srv = self._server_with_history(tmp, "not-json{")
|
||
self.assertIsNone(srv._turn_error_from_history("th1", "tu1"))
|
||
|
||
|
||
class StateVersionTest(unittest.TestCase):
|
||
def test_version_increments_and_conflict_logged(self):
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
path = os.path.join(tmp, "host-state.json")
|
||
s1 = {"tasks": {"a": {"status": "completed"}}, "automations": []}
|
||
host.save_state(path, s1)
|
||
with open(path) as f:
|
||
self.assertEqual(json.load(f)["version"], 1)
|
||
# 模拟另一写者：直接改磁盘版本为 5，再保存 s1（加载版本是 1）
|
||
with open(path) as f:
|
||
disk = json.load(f)
|
||
disk["version"] = 5
|
||
disk["tasks"]["b"] = {"status": "completed"}
|
||
with open(path, "w") as f:
|
||
json.dump(disk, f)
|
||
host.save_state(path, s1)
|
||
with open(path) as f:
|
||
merged = json.load(f)
|
||
self.assertEqual(merged["version"], 6)
|
||
self.assertIn("b", merged["tasks"])
|
||
self.assertIn("a", merged["tasks"])
|
||
log = Path(tmp, "host-run.log").read_text()
|
||
self.assertIn("STATE-CONFLICT", log)
|
||
|
||
|
||
class ServerRestartTest(unittest.TestCase):
|
||
@unittest.skipUnless(shutil.which("codex"),
|
||
"codex binary not on PATH (e.g. GitHub runner)")
|
||
def test_kill_then_ensure_alive(self):
|
||
# 不跑模型：只验证 server 进程死了能重拉、list 能通。
|
||
# 用配对二进制（VM 本机补丁），隔离 CODEX_HOME。
|
||
# Runner 上没有 codex 二进制时跳过，而不是失败。
|
||
import host as host_mod
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
codex_home = os.path.join(tmp, "codex-home")
|
||
srv = host_mod.Server(codex_home=codex_home, approval="deny")
|
||
try:
|
||
srv.initialize()
|
||
self.assertTrue(srv.is_alive())
|
||
srv.proc.kill()
|
||
srv.proc.wait(timeout=10)
|
||
self.assertFalse(srv.is_alive())
|
||
self.assertTrue(srv.ensure_alive())
|
||
self.assertTrue(srv.is_alive())
|
||
# list_threads 应能通（空或已有 thread 列表均可）
|
||
srv.list_threads()
|
||
finally:
|
||
try:
|
||
srv.close()
|
||
except Exception:
|
||
pass
|
||
|
||
|
||
class HealthRestoreRecoverTest(unittest.TestCase):
|
||
class FakeServer:
|
||
def initialize(self):
|
||
return {}
|
||
|
||
def list_threads(self):
|
||
return {"data": []}
|
||
|
||
def close(self):
|
||
pass
|
||
|
||
def test_corrupt_state_detected_and_restored_from_bak(self):
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
path = os.path.join(tmp, "host-state.json")
|
||
good = {"tasks": {"a": {"status": "completed"}},
|
||
"automations": [], "version": 3}
|
||
with open(path + ".bak", "w") as f:
|
||
json.dump(good, f)
|
||
with open(path, "w") as f:
|
||
f.write("{not json!!!")
|
||
strict, err = host.load_state_strict(path)
|
||
self.assertIsNone(strict)
|
||
self.assertIsNotNone(err)
|
||
ok, msg = host.restore_state(path)
|
||
self.assertTrue(ok, msg)
|
||
with open(path) as f:
|
||
restored = json.load(f)
|
||
self.assertEqual(restored["version"], 3)
|
||
self.assertTrue(os.path.exists(path + ".corrupt.") or
|
||
any(fn.startswith("host-state.json.corrupt.")
|
||
for fn in os.listdir(tmp)))
|
||
|
||
def test_restore_without_backup_fails_without_overwriting(self):
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
path = os.path.join(tmp, "host-state.json")
|
||
with open(path, "w") as f:
|
||
f.write("{bad")
|
||
ok, msg = host.restore_state(path)
|
||
self.assertFalse(ok)
|
||
# 坏文件被改名留底，主文件已不在（不是被空 state 覆盖）
|
||
self.assertFalse(os.path.exists(path))
|
||
|
||
def test_recover_marks_old_running_only(self):
|
||
import time as _time
|
||
now = _time.time()
|
||
state = {"tasks": {
|
||
"old": {"status": "running", "created_at": now - 7200,
|
||
"last_turn_at": now - 7200},
|
||
"fresh": {"status": "running", "created_at": now - 60,
|
||
"last_turn_at": now - 60},
|
||
"done": {"status": "completed", "created_at": now - 7200},
|
||
}, "automations": []}
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
path = os.path.join(tmp, "host-state.json")
|
||
recovered = host.recover_interrupted(state, path,
|
||
older_than=3600, cfg={})
|
||
self.assertEqual(recovered, ["old"])
|
||
self.assertEqual(state["tasks"]["old"]["status"], "interrupted")
|
||
self.assertEqual(state["tasks"]["fresh"]["status"], "running")
|
||
|
||
def test_health_with_fake_server(self):
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
path = os.path.join(tmp, "host-state.json")
|
||
with open(path, "w") as f:
|
||
json.dump({"tasks": {}, "automations": [], "version": 1}, f)
|
||
with open(path + ".bak", "w") as f:
|
||
json.dump({"tasks": {}, "automations": [], "version": 1}, f)
|
||
report = host.run_health_check(
|
||
path, {}, "/bin/true", tmp, "deny",
|
||
server_factory=lambda: self.FakeServer())
|
||
self.assertTrue(report["ok"], report)
|
||
self.assertTrue(report["checks"]["state_valid"])
|
||
self.assertTrue(report["checks"]["backup_exists"])
|
||
self.assertTrue(report["checks"]["server_ok"])
|
||
|
||
|
||
class GithubPRTest(unittest.TestCase):
|
||
class FakeGh:
|
||
def __init__(self, checks_output="", pr_url=""):
|
||
self.checks_output = checks_output
|
||
self.pr_url = pr_url
|
||
self.calls = []
|
||
|
||
def __call__(self, args):
|
||
self.calls.append(list(args))
|
||
import types
|
||
if args[:2] == ["pr", "checks"]:
|
||
return types.SimpleNamespace(returncode=0,
|
||
stdout=self.checks_output,
|
||
stderr="")
|
||
if args[:2] == ["pr", "create"]:
|
||
return types.SimpleNamespace(returncode=0,
|
||
stdout=self.pr_url + "\n",
|
||
stderr="")
|
||
if args[:2] == ["pr", "merge"]:
|
||
return types.SimpleNamespace(returncode=0,
|
||
stdout="Merged\n", stderr="")
|
||
return types.SimpleNamespace(returncode=1, stdout="",
|
||
stderr="unknown fake gh call")
|
||
|
||
def test_workflow_content_and_idempotent(self):
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
changed, path = host.generate_github_workflow(tmp)
|
||
self.assertTrue(changed)
|
||
content = Path(path).read_text()
|
||
self.assertIn("actions/checkout", content)
|
||
self.assertIn("setup-python", content)
|
||
self.assertIn("python3 -m unittest discover -s tests -v",
|
||
content)
|
||
changed2, _ = host.generate_github_workflow(tmp)
|
||
self.assertFalse(changed2)
|
||
# 内容被改坏后再次生成应恢复
|
||
Path(path).write_text("garbage\n")
|
||
changed3, _ = host.generate_github_workflow(tmp)
|
||
self.assertTrue(changed3)
|
||
self.assertEqual(Path(path).read_text(), content)
|
||
|
||
def test_pr_checks_parse_pass_fail_mixed(self):
|
||
checks, all_pass = host.parse_pr_checks(
|
||
"build\tpass\t1m\ntest\tpass\t2m\n")
|
||
self.assertTrue(all_pass)
|
||
self.assertEqual([c["state"] for c in checks], ["pass", "pass"])
|
||
checks, all_pass = host.parse_pr_checks(
|
||
"✓ build pass\n✗ test fail\n")
|
||
self.assertFalse(all_pass)
|
||
self.assertEqual([c["name"] for c in checks], ["build", "test"])
|
||
checks, all_pass = host.parse_pr_checks("")
|
||
self.assertFalse(all_pass)
|
||
self.assertEqual(checks, [])
|
||
|
||
def test_pr_merge_refuses_when_not_all_pass(self):
|
||
fake = self.FakeGh(checks_output="build\tpass\ntest\tfail\n")
|
||
task = {"name": "t", "pr_url": "https://example.com/pull/7",
|
||
"pr_number": 7, "status": "completed"}
|
||
ok, msg = host.merge_pr(task, gh_runner=fake)
|
||
self.assertFalse(ok)
|
||
self.assertIn("未全过", msg)
|
||
self.assertNotIn(["pr", "merge", "7", "--merge"], fake.calls)
|
||
self.assertEqual(task["status"], "completed")
|
||
|
||
def test_pr_merge_only_when_all_pass(self):
|
||
fake = self.FakeGh(checks_output="build\tpass\ntest\tpass\n")
|
||
task = {"name": "t", "pr_url": "https://example.com/pull/8",
|
||
"pr_number": 8, "status": "completed"}
|
||
ok, msg = host.merge_pr(task, gh_runner=fake)
|
||
self.assertTrue(ok, msg)
|
||
self.assertIn(["pr", "merge", "8", "--merge"], fake.calls)
|
||
self.assertEqual(task["status"], "merged")
|
||
|
||
def test_pr_without_branch_or_wrong_status_rejected(self):
|
||
with self.assertRaises(ValueError):
|
||
host.create_pr("/tmp", {"name": "t", "status": "completed"})
|
||
with self.assertRaises(ValueError):
|
||
host.create_pr("/tmp", {"name": "t", "branch": "host/t",
|
||
"status": "running"})
|
||
|
||
def test_pr_without_remote_gives_clear_error(self):
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
_init_git_repo(tmp)
|
||
Path(tmp, "f.txt").write_text("x\n")
|
||
subprocess.run(["git", "add", "-A"], cwd=tmp, check=True)
|
||
subprocess.run(["git", "-c", "user.name=Test", "-c", "user.email=test@example.com", "commit", "-qm", "initial"],
|
||
cwd=tmp, check=True)
|
||
task = {"name": "t", "branch": "host/t", "status": "completed",
|
||
"prompt": "do it"}
|
||
with self.assertRaises(RuntimeError) as ctx:
|
||
host.create_pr(tmp, task, gh_runner=self.FakeGh())
|
||
self.assertIn("remote", str(ctx.exception))
|
||
|
||
def test_pr_push_to_local_bare_remote_then_fake_gh(self):
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
repo = os.path.join(tmp, "repo")
|
||
os.makedirs(repo)
|
||
_init_git_repo(repo)
|
||
Path(repo, "f.txt").write_text("x\n")
|
||
subprocess.run(["git", "add", "-A"], cwd=repo, check=True)
|
||
subprocess.run(["git", "-c", "user.name=Test", "-c", "user.email=test@example.com", "commit", "-qm", "initial"],
|
||
cwd=repo, check=True)
|
||
subprocess.run(["git", "checkout", "-qb", "host/t1"],
|
||
cwd=repo, check=True)
|
||
Path(repo, "f.txt").write_text("y\n")
|
||
subprocess.run(["git", "-c", "user.name=Test", "-c", "user.email=test@example.com", "commit", "-qam", "change"],
|
||
cwd=repo, check=True)
|
||
remote = os.path.join(tmp, "remote.git")
|
||
subprocess.run(["git", "init", "--bare", "-q", remote],
|
||
check=True)
|
||
subprocess.run(["git", "remote", "add", "origin", remote],
|
||
cwd=repo, check=True)
|
||
fake = self.FakeGh(
|
||
pr_url="https://github.com/example/repo/pull/42")
|
||
task = {"name": "t1", "branch": "host/t1",
|
||
"status": "completed", "prompt": "change f"}
|
||
info = host.create_pr(repo, task, gh_runner=fake)
|
||
self.assertEqual(info["pr_number"], 42)
|
||
self.assertEqual(task["pr_url"],
|
||
"https://github.com/example/repo/pull/42")
|
||
self.assertEqual(task["pr_number"], 42)
|
||
branches = subprocess.run(
|
||
["git", "--git-dir", remote, "branch"], capture_output=True,
|
||
text=True).stdout
|
||
self.assertIn("host/t1", branches)
|
||
|
||
def test_pr_cli_without_remote_fails_with_clear_error(self):
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
repo = os.path.join(tmp, "repo")
|
||
os.makedirs(repo)
|
||
_init_git_repo(repo)
|
||
state_path = os.path.join(tmp, "host-state.json")
|
||
with open(state_path, "w") as f:
|
||
json.dump({"tasks": {"t1": {
|
||
"name": "t1", "branch": "host/t1",
|
||
"status": "completed", "prompt": "do it",
|
||
"repo": repo}}, "automations": []}, f)
|
||
cfg_path = os.path.join(tmp, "cfg.json")
|
||
with open(cfg_path, "w") as f:
|
||
json.dump({"repo": repo, "state": state_path}, f)
|
||
r = subprocess.run(
|
||
[sys.executable, str(REPO_ROOT / "host.py"),
|
||
"--config", cfg_path, "pr", "--task", "t1"],
|
||
capture_output=True, text=True)
|
||
self.assertNotEqual(r.returncode, 0)
|
||
self.assertIn("remote", r.stdout + r.stderr)
|
||
|
||
|
||
class TurnUsageSumTest(unittest.TestCase):
|
||
def _note(self, thread_id, turn_id, last):
|
||
return {"method": "thread/tokenUsage/updated",
|
||
"params": {"threadId": thread_id, "turnId": turn_id,
|
||
"tokenUsage": {"last": last}}}
|
||
|
||
def test_sums_last_across_completions(self):
|
||
# 2026-10-26 中继实测前两次通知：last 之和等于第二次的 total。
|
||
evs = [
|
||
self._note("t1", "u1", {
|
||
"inputTokens": 10906, "cachedInputTokens": 9192,
|
||
"cacheWriteInputTokens": 0, "outputTokens": 346,
|
||
"reasoningOutputTokens": 256, "totalTokens": 11252}),
|
||
self._note("t1", "u1", {
|
||
"inputTokens": 11041, "cachedInputTokens": 9192,
|
||
"cacheWriteInputTokens": 0, "outputTokens": 230,
|
||
"reasoningOutputTokens": 161, "totalTokens": 11271}),
|
||
]
|
||
self.assertEqual(host._sum_turn_usage(evs, "t1", "u1"), {
|
||
"input_tokens": 21947, "cached_input_tokens": 18384,
|
||
"cache_write_input_tokens": 0, "output_tokens": 576,
|
||
"reasoning_output_tokens": 417, "total_tokens": 22523,
|
||
"completions": 2})
|
||
|
||
def test_filters_by_thread_and_turn(self):
|
||
evs = [
|
||
self._note("t1", "u1", {
|
||
"inputTokens": 10, "cachedInputTokens": 0,
|
||
"cacheWriteInputTokens": 0, "outputTokens": 5,
|
||
"reasoningOutputTokens": 0, "totalTokens": 15}),
|
||
self._note("t1", "u2", {
|
||
"inputTokens": 999, "cachedInputTokens": 0,
|
||
"cacheWriteInputTokens": 0, "outputTokens": 999,
|
||
"reasoningOutputTokens": 0, "totalTokens": 1998}),
|
||
self._note("t2", "u1", {
|
||
"inputTokens": 999, "cachedInputTokens": 0,
|
||
"cacheWriteInputTokens": 0, "outputTokens": 999,
|
||
"reasoningOutputTokens": 0, "totalTokens": 1998}),
|
||
{"method": "item/completed", "params": {}},
|
||
]
|
||
usage = host._sum_turn_usage(evs, "t1", "u1")
|
||
self.assertEqual(usage["input_tokens"], 10)
|
||
self.assertEqual(usage["completions"], 1)
|
||
self.assertIsNone(host._sum_turn_usage(evs, "t1", "u9"))
|
||
self.assertIsNone(host._sum_turn_usage([], "t1", "u1"))
|
||
|
||
def test_retry_sums_usage_across_attempts(self):
|
||
class FakeServer:
|
||
def __init__(self, results):
|
||
self.results = list(results)
|
||
|
||
def run_turn(self, thread_id, prompt, model=None, effort=None,
|
||
sandbox=None):
|
||
return self.results.pop(0)
|
||
|
||
task = {"name": "t", "thread_id": "x", "sandbox": None,
|
||
"max_retries": 1, "retry_count": 0, "turns": []}
|
||
state = {"tasks": {"t": task}, "automations": []}
|
||
srv = FakeServer([
|
||
{"status": "failed", "text": "", "usage": {
|
||
"input_tokens": 100, "cached_input_tokens": 0,
|
||
"cache_write_input_tokens": 0, "output_tokens": 10,
|
||
"reasoning_output_tokens": 0, "total_tokens": 110,
|
||
"completions": 1}},
|
||
{"status": "completed", "text": "ok", "usage": {
|
||
"input_tokens": 200, "cached_input_tokens": 50,
|
||
"cache_write_input_tokens": 0, "output_tokens": 20,
|
||
"reasoning_output_tokens": 5, "total_tokens": 220,
|
||
"completions": 1}},
|
||
])
|
||
result = host.run_task_turn_with_retry(
|
||
srv, state, task, "do it", "m", "low", cfg={})
|
||
self.assertEqual(result["status"], "completed")
|
||
self.assertEqual(result["usage"]["input_tokens"], 300)
|
||
self.assertEqual(result["usage"]["cached_input_tokens"], 50)
|
||
self.assertEqual(result["usage"]["total_tokens"], 330)
|
||
self.assertEqual(result["usage"]["completions"], 2)
|
||
|
||
|
||
class LongTurnAlignmentTest(unittest.TestCase):
|
||
def test_partial_text_collects_agent_messages_only(self):
|
||
events = [
|
||
{"method": "item/completed",
|
||
"params": {"item": {"type": "agentMessage",
|
||
"text": "first part"}}},
|
||
{"method": "item/agentMessage/delta",
|
||
"params": {"delta": "should be ignored"}},
|
||
{"method": "item/completed",
|
||
"params": {"item": {"type": "toolCall",
|
||
"text": "not a message"}}},
|
||
{"method": "item/completed",
|
||
"params": {"item": {"type": "agentMessage",
|
||
"text": "second part"}}},
|
||
]
|
||
self.assertEqual(host._partial_text_from_events(events),
|
||
"first part\nsecond part")
|
||
self.assertEqual(host._partial_text_from_events([]), "")
|
||
|
||
def test_timeout_maps_to_interrupted_and_threads_timeout(self):
|
||
class FakeServer:
|
||
def __init__(self):
|
||
self.seen_timeout = None
|
||
|
||
def run_turn(self, thread_id, prompt, model=None, effort=None,
|
||
sandbox=None, timeout=None):
|
||
self.seen_timeout = timeout
|
||
return {"status": "timeout", "text": "partial",
|
||
"turn_id": "u1"}
|
||
|
||
task = {"name": "t", "thread_id": "x", "sandbox": None,
|
||
"max_retries": 0, "retry_count": 0, "turns": []}
|
||
state = {"tasks": {"t": task}, "automations": []}
|
||
srv = FakeServer()
|
||
result = host.run_task_turn(srv, state, task, "do it", "m", "low",
|
||
timeout=0)
|
||
self.assertEqual(result["status"], "timeout")
|
||
self.assertEqual(task["status"], "interrupted")
|
||
self.assertEqual(task["last_text"], "partial")
|
||
self.assertEqual(srv.seen_timeout, 0)
|
||
|
||
def test_timeout_none_keeps_old_fake_server_working(self):
|
||
class OldFakeServer:
|
||
def run_turn(self, thread_id, prompt, model=None, effort=None,
|
||
sandbox=None):
|
||
return {"status": "completed", "text": "ok"}
|
||
|
||
task = {"name": "t", "thread_id": "x", "sandbox": None,
|
||
"max_retries": 0, "retry_count": 0, "turns": []}
|
||
state = {"tasks": {"t": task}, "automations": []}
|
||
result = host.run_task_turn(OldFakeServer(), state, task, "do it",
|
||
"m", "low")
|
||
self.assertEqual(result["status"], "completed")
|
||
self.assertEqual(task["status"], "completed")
|
||
|
||
def test_interrupt_turn_sends_thread_and_turn_ids(self):
|
||
srv = object.__new__(host.Server)
|
||
seen = {}
|
||
|
||
def fake_request(method, params=None, timeout=120):
|
||
seen["method"] = method
|
||
seen["params"] = params
|
||
seen["timeout"] = timeout
|
||
return {"result": {}}
|
||
|
||
srv.request = fake_request
|
||
self.assertEqual(srv.interrupt_turn("th1", "tu1"),
|
||
{"result": {}})
|
||
self.assertEqual(seen["method"], "turn/interrupt")
|
||
self.assertEqual(seen["params"],
|
||
{"threadId": "th1", "turnId": "tu1"})
|
||
self.assertIsNone(srv.interrupt_turn("th1", None))
|
||
self.assertIsNone(srv.interrupt_turn(None, "tu1"))
|
||
|
||
|
||
class GoalLoopLongTaskTest(unittest.TestCase):
|
||
def _task(self, tmp):
|
||
return {"name": "g", "thread_id": "x", "cwd": tmp, "sandbox": None,
|
||
"max_retries": 0, "retry_count": 0, "turns": []}
|
||
|
||
def test_goal_met_on_first_round_checkpoints(self):
|
||
class FakeServer:
|
||
def run_turn(self, thread_id, prompt, model=None, effort=None,
|
||
sandbox=None, timeout=None):
|
||
return {"status": "completed", "text": "done",
|
||
"turn_id": "u1"}
|
||
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
state_path = os.path.join(tmp, "host-state.json")
|
||
state = {"tasks": {}, "automations": []}
|
||
task = self._task(tmp)
|
||
state["tasks"]["g"] = task
|
||
result = host.run_goal_loop(FakeServer(), state, state_path,
|
||
task, "make it", "true", 3, "m",
|
||
"low", None, cfg={}, timeout=600)
|
||
self.assertTrue(result["ok"])
|
||
self.assertEqual(task["status"], "goal-met")
|
||
self.assertEqual(task["goal_rounds_done"], 1)
|
||
self.assertEqual(task["goal"], "make it")
|
||
self.assertEqual(task["verify_command"], "true")
|
||
saved = host.load_state(state_path)
|
||
self.assertEqual(saved["tasks"]["g"]["status"], "goal-met")
|
||
|
||
def test_goal_unmet_after_max_rounds_and_timeout_feedback(self):
|
||
class FakeServer:
|
||
def __init__(self):
|
||
self.prompts = []
|
||
|
||
def run_turn(self, thread_id, prompt, model=None, effort=None,
|
||
sandbox=None, timeout=None):
|
||
self.prompts.append(prompt)
|
||
if len(self.prompts) == 1:
|
||
return {"status": "timeout", "text": "partial work",
|
||
"turn_id": "u1"}
|
||
return {"status": "completed", "text": "done",
|
||
"turn_id": "u2"}
|
||
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
state_path = os.path.join(tmp, "host-state.json")
|
||
state = {"tasks": {}, "automations": []}
|
||
task = self._task(tmp)
|
||
state["tasks"]["g"] = task
|
||
srv = FakeServer()
|
||
result = host.run_goal_loop(srv, state, state_path,
|
||
task, "make it", "false", 2, "m",
|
||
"low", None, cfg={}, timeout=600)
|
||
self.assertFalse(result["ok"])
|
||
self.assertEqual(task["status"], "goal-unmet")
|
||
self.assertEqual(task["goal_rounds_done"], 2)
|
||
# 第二轮的提示必须说清是回合中断，不是验收失败。
|
||
self.assertIn("timeout", srv.prompts[1])
|
||
self.assertIn("未完成", srv.prompts[1])
|
||
saved = host.load_state(state_path)
|
||
self.assertEqual(saved["tasks"]["g"]["goal_rounds_done"], 2)
|
||
|
||
def test_goal_consecutive_failed_turns_stop_spinning(self):
|
||
# 不限轮次 + 瞬间失败轮次会无限空转（2026-10-29 实弹：
|
||
# 断点后 274 轮全失败）；3 轮连续失败即中断出口。
|
||
class FakeServer:
|
||
def __init__(self):
|
||
self.calls = 0
|
||
|
||
def run_turn(self, thread_id, prompt, model=None, effort=None,
|
||
sandbox=None, timeout=None):
|
||
self.calls += 1
|
||
return {"status": "failed", "text": "",
|
||
"turn_id": "f%d" % self.calls}
|
||
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
state_path = os.path.join(tmp, "host-state.json")
|
||
state = {"tasks": {}, "automations": []}
|
||
task = self._task(tmp)
|
||
state["tasks"]["g"] = task
|
||
srv = FakeServer()
|
||
result = host.run_goal_loop(srv, state, state_path,
|
||
task, "make it", None, 0, "m",
|
||
"low", None, cfg={}, timeout=600)
|
||
self.assertEqual(srv.calls, 3)
|
||
self.assertTrue(result["interrupted"])
|
||
self.assertEqual(task["status"], "interrupted")
|
||
self.assertIn("连续 3 轮", task["goal_interrupted_reason"])
|
||
|
||
def test_goal_resume_continues_from_checkpoint(self):
|
||
class FakeServer:
|
||
def __init__(self):
|
||
self.prompts = []
|
||
|
||
def run_turn(self, thread_id, prompt, model=None, effort=None,
|
||
sandbox=None, timeout=None):
|
||
self.prompts.append(prompt)
|
||
return {"status": "completed", "text": "done",
|
||
"turn_id": "u9"}
|
||
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
state_path = os.path.join(tmp, "host-state.json")
|
||
state = {"tasks": {}, "automations": []}
|
||
task = self._task(tmp)
|
||
state["tasks"]["g"] = task
|
||
srv = FakeServer()
|
||
# 模拟被杀在检查点后的任务：已跑 1 轮、验测失败、状态 goal-running。
|
||
task["status"] = "goal-running"
|
||
task["goal"] = "make it"
|
||
task["verify_command"] = "true"
|
||
task["goal_max_rounds"] = 3
|
||
task["goal_rounds_done"] = 1
|
||
task["goal_last_verify"] = "missing file"
|
||
task["goal_last_turn_status"] = "completed"
|
||
task["turns"] = [{"prompt": "make it",
|
||
"result": {"status": "completed",
|
||
"text": "partial"}}]
|
||
result = host.run_goal_loop(srv, state, state_path, task,
|
||
"make it", "true", 3, "m", "low",
|
||
None, cfg={}, timeout=600,
|
||
start_round=2,
|
||
last_verify="missing file",
|
||
last_turn_status="completed")
|
||
self.assertTrue(result["ok"])
|
||
self.assertEqual(task["status"], "goal-met")
|
||
self.assertEqual(task["goal_rounds_done"], 2)
|
||
self.assertIn("missing file", srv.prompts[-1])
|
||
|
||
def test_goal_fields_saved_before_first_turn(self):
|
||
class FakeServer:
|
||
def run_turn(self, thread_id, prompt, model=None, effort=None,
|
||
sandbox=None, timeout=None):
|
||
raise RuntimeError("killed during turn")
|
||
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
state_path = os.path.join(tmp, "host-state.json")
|
||
state = {"tasks": {}, "automations": []}
|
||
task = self._task(tmp)
|
||
state["tasks"]["g"] = task
|
||
# 2026-10-28 起：回合抛错不再向上传播（旧断言），而
|
||
# 是记 interrupted + 标记行（云端可断点续跑）；首轮
|
||
# 前落盘的字段仍必须都在。
|
||
result = host.run_goal_loop(FakeServer(), state, state_path,
|
||
task, "make it", "true", 3, "m",
|
||
"low", None, cfg={}, timeout=600)
|
||
self.assertTrue(result["interrupted"])
|
||
saved = host.load_state(state_path)
|
||
saved_task = saved["tasks"]["g"]
|
||
self.assertEqual(saved_task["goal"], "make it")
|
||
self.assertEqual(saved_task["verify_command"], "true")
|
||
self.assertEqual(saved_task["goal_rounds_done"], 0)
|
||
self.assertEqual(saved_task["status"], "interrupted")
|
||
self.assertIn("killed during turn",
|
||
saved_task["goal_interrupted_reason"])
|
||
|
||
def test_goal_budget_stop_is_interrupted_not_unmet(self):
|
||
class FakeServer:
|
||
def run_turn(self, thread_id, prompt, model=None, effort=None,
|
||
sandbox=None, timeout=None):
|
||
raise AssertionError("budget stop must not run a turn")
|
||
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
state_path = os.path.join(tmp, "host-state.json")
|
||
state = {"tasks": {}, "automations": []}
|
||
task = self._task(tmp)
|
||
state["tasks"]["g"] = task
|
||
today = __import__("time").strftime("%Y-%m-%d")
|
||
state["usage"] = {"date": today, "turns": 5, "failed_turns": 0}
|
||
result = host.run_goal_loop(FakeServer(), state, state_path,
|
||
task, "make it", "true", 3, "m",
|
||
"low", None,
|
||
cfg={"max_daily_turns": 5},
|
||
timeout=600)
|
||
self.assertTrue(result["budget_stopped"])
|
||
self.assertFalse(result["ok"])
|
||
self.assertEqual(task["status"], "interrupted")
|
||
|
||
def test_goal_verify_timeout_is_a_verify_failure(self):
|
||
class FakeServer:
|
||
def run_turn(self, thread_id, prompt, model=None, effort=None,
|
||
sandbox=None, timeout=None):
|
||
return {"status": "completed", "text": "done",
|
||
"turn_id": "u1"}
|
||
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
state_path = os.path.join(tmp, "host-state.json")
|
||
state = {"tasks": {}, "automations": []}
|
||
task = self._task(tmp)
|
||
state["tasks"]["g"] = task
|
||
result = host.run_goal_loop(FakeServer(), state, state_path,
|
||
task, "make it", "sleep 5", 1, "m",
|
||
"low", None, cfg={}, timeout=600,
|
||
verify_timeout=1)
|
||
self.assertFalse(result["ok"])
|
||
self.assertEqual(task["status"], "goal-unmet")
|
||
self.assertIn("verify timeout", task["goal_last_verify"])
|
||
|
||
def test_goal_llm_decided_met_on_marker_unlimited(self):
|
||
# 新形态：verify=None、max_rounds=0（不限轮次）；首轮提示
|
||
# 含标记约定；第 2 轮模型输出标记行即达标。
|
||
class FakeServer:
|
||
def __init__(self):
|
||
self.prompts = []
|
||
|
||
def run_turn(self, thread_id, prompt, model=None, effort=None,
|
||
sandbox=None, timeout=None):
|
||
self.prompts.append(prompt)
|
||
if len(self.prompts) == 1:
|
||
return {"status": "completed", "text": "working",
|
||
"turn_id": "u1"}
|
||
return {"status": "completed",
|
||
"text": "done\n目标已达成",
|
||
"turn_id": "u2"}
|
||
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
state_path = os.path.join(tmp, "host-state.json")
|
||
state = {"tasks": {}, "automations": []}
|
||
task = self._task(tmp)
|
||
state["tasks"]["g"] = task
|
||
srv = FakeServer()
|
||
result = host.run_goal_loop(srv, state, state_path,
|
||
task, "make it", None, 0, "m",
|
||
"low", None, cfg={}, timeout=600)
|
||
self.assertTrue(result["ok"])
|
||
self.assertEqual(task["status"], "goal-met")
|
||
self.assertEqual(task["goal_rounds_done"], 2)
|
||
self.assertEqual(task["goal_max_rounds"], 0)
|
||
self.assertIn("目标已达成", srv.prompts[0])
|
||
self.assertIn("目标已达成", srv.prompts[1])
|
||
|
||
def test_goal_marker_trailing_punctuation_counts(self):
|
||
# 2026-10-27 实弹教训：提示让模型输出“目标已达成。”（带
|
||
# 句号），检测若要求整行恰好等于无句号标记，循环空转
|
||
# 292 轮。行尾句末标点必须容忍，但单独一行要求不动。
|
||
self.assertTrue(host._llm_declared_done("done\n目标已达成"))
|
||
self.assertTrue(host._llm_declared_done("done\n目标已达成。"))
|
||
self.assertTrue(host._llm_declared_done("done\n目标已达成！"))
|
||
self.assertTrue(host._llm_declared_done("done\n目标已达成 。"))
|
||
self.assertTrue(host._llm_declared_done("done\n目标已达成。 "))
|
||
self.assertFalse(host._llm_declared_done("done\nxx目标已达成"))
|
||
self.assertFalse(host._llm_declared_done("done\n目标已达成，继续"))
|
||
self.assertFalse(host._llm_declared_done("done\n目标已达成吗？"))
|
||
self.assertFalse(host._llm_declared_done(""))
|
||
self.assertFalse(host._llm_declared_done(None))
|
||
|
||
def test_goal_llm_decided_marker_ignored_on_timeout_turn(self):
|
||
# 超时回合的部分文本即使含标记行也不算达标，必须继续跑。
|
||
class FakeServer:
|
||
def __init__(self):
|
||
self.calls = 0
|
||
|
||
def run_turn(self, thread_id, prompt, model=None, effort=None,
|
||
sandbox=None, timeout=None):
|
||
self.calls += 1
|
||
if self.calls == 1:
|
||
return {"status": "timeout",
|
||
"text": "partial\n目标已达成",
|
||
"turn_id": "u1"}
|
||
return {"status": "completed",
|
||
"text": "done\n目标已达成",
|
||
"turn_id": "u2"}
|
||
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
state_path = os.path.join(tmp, "host-state.json")
|
||
state = {"tasks": {}, "automations": []}
|
||
task = self._task(tmp)
|
||
state["tasks"]["g"] = task
|
||
result = host.run_goal_loop(FakeServer(), state, state_path,
|
||
task, "make it", None, 0, "m",
|
||
"low", None, cfg={}, timeout=600)
|
||
self.assertTrue(result["ok"])
|
||
self.assertEqual(task["status"], "goal-met")
|
||
self.assertEqual(task["goal_rounds_done"], 2)
|
||
|
||
def test_goal_llm_decided_unlimited_past_three_rounds(self):
|
||
# 0 = 不限轮次：第 4 轮才宣告也照常结束，不会被旧的
|
||
# 默认 3 轮上限截断。
|
||
class FakeServer:
|
||
def __init__(self):
|
||
self.calls = 0
|
||
|
||
def run_turn(self, thread_id, prompt, model=None, effort=None,
|
||
sandbox=None, timeout=None):
|
||
self.calls += 1
|
||
if self.calls < 4:
|
||
return {"status": "completed", "text": "working",
|
||
"turn_id": "u%d" % self.calls}
|
||
return {"status": "completed",
|
||
"text": "done\n目标已达成",
|
||
"turn_id": "u4"}
|
||
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
state_path = os.path.join(tmp, "host-state.json")
|
||
state = {"tasks": {}, "automations": []}
|
||
task = self._task(tmp)
|
||
state["tasks"]["g"] = task
|
||
result = host.run_goal_loop(FakeServer(), state, state_path,
|
||
task, "make it", None, 0, "m",
|
||
"low", None, cfg={}, timeout=600)
|
||
self.assertTrue(result["ok"])
|
||
self.assertEqual(task["status"], "goal-met")
|
||
self.assertEqual(task["goal_rounds_done"], 4)
|
||
|
||
|
||
|
||
class ProgressInterruptTest(unittest.TestCase):
|
||
"""长等待流式进度 + 跨进程中断请求：只测可单测的等待循环与
|
||
路径/透传，不跑模型、不拉 app-server。"""
|
||
|
||
def test_wait_turn_completed(self):
|
||
import queue
|
||
import threading
|
||
|
||
q = queue.Queue()
|
||
|
||
def _put():
|
||
import time as _time
|
||
_time.sleep(0.05)
|
||
q.put({"turn": "done"})
|
||
|
||
threading.Thread(target=_put, daemon=True).start()
|
||
status, done = host._wait_turn(q, timeout=5)
|
||
self.assertEqual(status, "completed")
|
||
self.assertEqual(done, {"turn": "done"})
|
||
|
||
def test_wait_turn_timeout(self):
|
||
import queue
|
||
import time as _time
|
||
|
||
q = queue.Queue()
|
||
start = _time.monotonic()
|
||
status, done = host._wait_turn(q, timeout=0.2,
|
||
progress_interval=10)
|
||
elapsed = _time.monotonic() - start
|
||
self.assertEqual(status, "timeout")
|
||
self.assertIsNone(done)
|
||
self.assertGreaterEqual(elapsed, 0.2)
|
||
self.assertLess(elapsed, 1.0)
|
||
|
||
def test_wait_turn_interrupt_request(self):
|
||
import queue
|
||
|
||
q = queue.Queue()
|
||
calls = []
|
||
|
||
def _check():
|
||
calls.append(1)
|
||
return len(calls) >= 2
|
||
|
||
status, done = host._wait_turn(q, timeout=5,
|
||
interrupt_check=_check)
|
||
self.assertEqual(status, "interrupted-requested")
|
||
self.assertIsNone(done)
|
||
|
||
def test_wait_turn_progress_calls(self):
|
||
import queue
|
||
|
||
q = queue.Queue()
|
||
seen = []
|
||
status, _ = host._wait_turn(q, timeout=0.35,
|
||
progress_interval=0.1,
|
||
on_progress=seen.append)
|
||
self.assertEqual(status, "timeout")
|
||
self.assertGreaterEqual(len(seen), 2)
|
||
self.assertEqual(seen, sorted(seen))
|
||
|
||
def test_interrupt_request_path(self):
|
||
path = host._interrupt_request_path("/tmp/abc/host-state.json",
|
||
"my task/a")
|
||
self.assertEqual(path, "/tmp/abc/interrupt-requests/my-task-a.json")
|
||
|
||
def test_run_task_turn_passes_interrupt_file(self):
|
||
class RecordingServer:
|
||
def __init__(self):
|
||
self.seen = None
|
||
|
||
def run_turn(self, thread_id, prompt, model=None, effort=None,
|
||
sandbox=None, timeout=None, interrupt_file=None):
|
||
self.seen = interrupt_file
|
||
return {"status": "interrupted", "text": "part",
|
||
"turn_id": "t1"}
|
||
|
||
srv = RecordingServer()
|
||
task = {"name": "g", "thread_id": "x", "sandbox": None,
|
||
"turns": []}
|
||
state = {"tasks": {"g": task}, "automations": []}
|
||
result = host.run_task_turn(srv, state, task, "do it", "m",
|
||
"low", interrupt_file="/tmp/req.json")
|
||
self.assertEqual(result["status"], "interrupted")
|
||
self.assertEqual(task["status"], "interrupted")
|
||
self.assertEqual(srv.seen, "/tmp/req.json")
|
||
|
||
def test_progress_line_is_parse_safe(self):
|
||
line = "turn progress: elapsed=30s"
|
||
self.assertNotIn("goal met", line)
|
||
self.assertNotIn("goal unmet", line)
|
||
self.assertNotIn("usage input=", line)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
unittest.main()
|
||
|