feat: 长任务补齐——流式进度 + 跨进程中断 + worker 超时约定
- 长等待流式进度：run_turn 改 1 秒切片轮询（_wait_turn），每满 --progress-interval（默认 30 秒，0 关闭，CLI > 配置 > 默认）往 stderr 打 turn progress 并往链路文件记 kind=progress；进度绝不走 stdout，不破坏 dispatch JSON 与 goal 结论行/用量行解析（经 cloud 真正则验过） - 跨进程中断：interrupt 直接不生效时写 <state_dir>/interrupt-requests/<task>.json，属主等 turn 时每秒轮询到后用自己的 live 连接发 turn/interrupt 并记 interrupted；10 秒内未响应也不删文件，属主下次轮询仍处理；接线覆盖 dispatch/run/goal/goal-resume/delegate/parallel/fix-review/run-due；旧 server 不接受 interrupt_file 时退回不传再跑 - cloud worker 超时约定：--task-timeout 0 归一化为 None（不限时，与 host turn_timeout 同约定，Docker 路径仍受 docker_timeout 约束），默认 1800 不变；超时仍记 failed（现有单测合同不动） 测试：host 56 全过（新增 7）、cloud 526 全过（4 跳过，新增 1 个 0=不限时）。
This commit is contained in:
host committed
1 parent
028ef02a86
commit
1128fd3076
6 files changed
+400
-34
No files matched your search
@@ -111,6 +111,75 @@ def _partial_text_from_events(events):
|
||||
return "\n".join(parts)
|
||||
|
||||
|
||||
def _interrupt_request_path(state_path, task_name):
|
||||
"""跨进程中断请求文件路径：<state_dir>/interrupt-requests/<task>.json。
|
||||
|
||||
属主进程在等 turn 时每秒轮询这个文件，另一个进程的
|
||||
`host.py interrupt --task` 直接协议中断不生效时就写这个文件，
|
||||
属主用自己的 live 连接发 turn/interrupt——只有属主的连接
|
||||
一定拥有这个 turn，这是跨进程中断可靠的原因。文件本身不
|
||||
由 run_task_turn 创建，只由 interrupt 命令写、属主读到后删。
|
||||
"""
|
||||
safe = re.sub(r"[^A-Za-z0-9_.-]", "-", str(task_name))
|
||||
return os.path.join(
|
||||
os.path.dirname(os.path.abspath(state_path)),
|
||||
"interrupt-requests", safe + ".json")
|
||||
|
||||
|
||||
def _wait_turn(q, timeout, progress_interval=30, on_progress=None,
|
||||
interrupt_check=None):
|
||||
"""等 turn/completed 的轮询等待（对齐 Codex App 的长等待语义）。
|
||||
|
||||
原来是一次性 q.get(timeout) / q.get()，长等待全程静默、另一
|
||||
进程无法插手；现在按 1 秒切片轮询，每片做两件事：
|
||||
1) interrupt_check() 为真时立刻返回 ("interrupted-requested", None)，
|
||||
由调用方用自己的 live 连接发 turn/interrupt 再记 interrupted；
|
||||
2) progress_interval > 0 且距上次进度已满一个间隔时调
|
||||
on_progress(elapsed_seconds)，让调用方打流式进度。
|
||||
返回：("completed", done) / ("timeout", None) /
|
||||
("interrupted-requested", None)。timeout 为 None 或 <=0 时
|
||||
不限时，只等 done 或中断请求；正数时到时返回 timeout。
|
||||
"""
|
||||
start = time.monotonic()
|
||||
next_progress = progress_interval if progress_interval and \
|
||||
progress_interval > 0 else None
|
||||
while True:
|
||||
elapsed = time.monotonic() - start
|
||||
if timeout and timeout > 0:
|
||||
remaining = timeout - elapsed
|
||||
if remaining <= 0:
|
||||
return "timeout", None
|
||||
slice_s = min(1.0, remaining)
|
||||
else:
|
||||
slice_s = 1.0
|
||||
# 切片还要受下一个进度点约束，否则一个长切片会把
|
||||
# 中间的进度检查全吞掉（timeout=0.35、间隔=0.1 时只会
|
||||
# 打 1 次进度，而不是 3 次；单测抓住了这个 bug）。
|
||||
if next_progress is not None:
|
||||
slice_s = min(slice_s, max(0.01, next_progress - elapsed))
|
||||
try:
|
||||
done = q.get(timeout=slice_s)
|
||||
return "completed", done
|
||||
except queue.Empty:
|
||||
pass
|
||||
elapsed = time.monotonic() - start
|
||||
if interrupt_check is not None:
|
||||
try:
|
||||
if interrupt_check():
|
||||
return "interrupted-requested", None
|
||||
except Exception:
|
||||
pass
|
||||
if next_progress is not None and elapsed >= next_progress:
|
||||
if on_progress is not None:
|
||||
try:
|
||||
on_progress(elapsed)
|
||||
except Exception:
|
||||
pass
|
||||
next_progress += progress_interval
|
||||
if timeout and timeout > 0 and elapsed >= timeout:
|
||||
return "timeout", None
|
||||
|
||||
|
||||
def _add_usage(a, b):
|
||||
"""两份用量相加（任一为空时返回另一份；都为空返回 None）。"""
|
||||
if not a:
|
||||
@@ -202,10 +271,14 @@ def _summarize_trace_item(method, params):
|
||||
class Server:
|
||||
"""单个 app-server 进程，多 thread 并发。"""
|
||||
|
||||
def __init__(self, codex_bin=None, codex_home=None, approval="deny"):
|
||||
def __init__(self, codex_bin=None, codex_home=None, approval="deny",
|
||||
progress_interval=30):
|
||||
self.codex_bin = codex_bin or default_codex_bin()
|
||||
self.codex_home = codex_home
|
||||
self.approval = approval
|
||||
# 流式进度间隔（秒）：run_turn 等待时每满一个间隔打一次
|
||||
# 进度；0 表示关闭进度输出，但中断轮询仍每秒进行。
|
||||
self.progress_interval = progress_interval
|
||||
self.send_lock = threading.Lock()
|
||||
self.next_id = 1
|
||||
self.pending = {}
|
||||
@@ -432,7 +505,7 @@ class Server:
|
||||
|
||||
def run_turn(self, thread_id, prompt, model=None, effort=None,
|
||||
parent_turn_id=None, root_turn_id=None, timeout=600,
|
||||
sandbox=None):
|
||||
sandbox=None, progress_interval=None, interrupt_file=None):
|
||||
# 开 turn 前先看 server 是否还活着，死了就重拉并 resume
|
||||
# 这个 thread（thread 持久化在 CODEX_HOME 里，重拉不丢），
|
||||
# 否则 turn/start 会直接 BrokenPipe 失败。
|
||||
@@ -477,10 +550,82 @@ class Server:
|
||||
# 或模型自己结束）；正数才按秒计时。默认仍是 600，
|
||||
# 由调用方（CLI --turn-timeout / 配置 turn_timeout）
|
||||
# 显式改，不改默认就不会悄悄挂起自动化。
|
||||
if timeout and timeout > 0:
|
||||
done = q.get(timeout=timeout)
|
||||
else:
|
||||
done = q.get()
|
||||
# 等待不再是一次性 q.get()：按 1 秒切片轮询，每满
|
||||
# progress_interval 打一次流式进度（stderr，绝不
|
||||
# 走 stdout——dispatch/run 的 stdout 是 JSON、goal
|
||||
# 的 stdout 要被 worker 解析结论行与用量行，进度
|
||||
# 进 stdout 会把这两种解析都打坏；链路文件里也
|
||||
# 同步记一条 kind=progress，console 按 fallback
|
||||
# 把它当 JSON 行展示），并每秒检查跨进程中断
|
||||
# 请求文件（见 _interrupt_request_path）。
|
||||
interval = self.progress_interval if \
|
||||
progress_interval is None else progress_interval
|
||||
|
||||
def _on_progress(elapsed):
|
||||
sys.stderr.write(
|
||||
"turn progress: elapsed=%ds\n"
|
||||
% int(elapsed))
|
||||
sys.stderr.flush()
|
||||
if self.trace_path is not None and \
|
||||
self._trace_lines < TRACE_MAX_LINES:
|
||||
try:
|
||||
with self.events_lock:
|
||||
n_events = len(self.events)
|
||||
with open(self.trace_path, "a",
|
||||
encoding="utf-8") as f:
|
||||
f.write(json.dumps(
|
||||
{"kind": "progress",
|
||||
"elapsed_s": int(elapsed),
|
||||
"events": n_events},
|
||||
ensure_ascii=False) + "\n")
|
||||
self._trace_lines += 1
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def _interrupt_check():
|
||||
return bool(interrupt_file) and \
|
||||
os.path.exists(interrupt_file)
|
||||
|
||||
wait_status, done = _wait_turn(
|
||||
q, timeout, progress_interval=interval,
|
||||
on_progress=_on_progress,
|
||||
interrupt_check=_interrupt_check)
|
||||
if wait_status == "interrupted-requested":
|
||||
# 属主用自己的 live 连接发中断：只有这个连接
|
||||
# 一定拥有该 turn，另一进程的直接中断未必生效。
|
||||
# 中断请求本身已明确，属主不再继续等：即使
|
||||
# 服务端回 error 也记 interrupted（任务可续跑，
|
||||
# 不是失败），并把请求文件删掉避免重复触发；
|
||||
# 服务端侧的停止是 best-effort，stderr 留一行。
|
||||
try:
|
||||
resp = self.interrupt_turn(thread_id, turn_id)
|
||||
if resp and "error" in resp:
|
||||
sys.stderr.write(
|
||||
"turn interrupt request returned error: "
|
||||
"%s\n" % (resp,))
|
||||
sys.stderr.flush()
|
||||
except Exception as exc:
|
||||
sys.stderr.write(
|
||||
"turn interrupt request failed: %s\n" % (exc,))
|
||||
sys.stderr.flush()
|
||||
try:
|
||||
if interrupt_file:
|
||||
os.remove(interrupt_file)
|
||||
except Exception:
|
||||
pass
|
||||
with self.events_lock:
|
||||
evs = list(self.events[start_events:])
|
||||
return {"status": "interrupted",
|
||||
"text": _partial_text_from_events(evs),
|
||||
"turn_id": turn_id,
|
||||
"usage": _sum_turn_usage(evs, thread_id, turn_id)}
|
||||
if wait_status == "timeout":
|
||||
with self.events_lock:
|
||||
evs = list(self.events[start_events:])
|
||||
return {"status": "timeout",
|
||||
"text": _partial_text_from_events(evs),
|
||||
"turn_id": turn_id,
|
||||
"usage": _sum_turn_usage(evs, thread_id, turn_id)}
|
||||
except queue.Empty:
|
||||
with self.events_lock:
|
||||
evs = list(self.events[start_events:])
|
||||
@@ -534,9 +679,11 @@ class Server:
|
||||
def interrupt_turn(self, thread_id, turn_id):
|
||||
"""对齐 Codex App 的停止按钮：向 app-server 发 turn/interrupt。
|
||||
协议要求 threadId + turnId；turn_id 缺失时无法中断，返回 None。
|
||||
注意：中断只对本 server 进程正在跑的 turn 可靠，另一进程的
|
||||
turn 未必能被本进程的中断请求停下，此时应直接 Ctrl-C
|
||||
等待中的那个进程（其 finally 会清队列，recover 会标 interrupted）。"""
|
||||
注意：直接中断只对本 server 进程正在跑的 turn 可靠，另一进程的
|
||||
turn 未必能被本进程的中断请求停下；跨进程的可靠路径是
|
||||
中断请求文件（见 _interrupt_request_path）：interrupt 命令
|
||||
直接中断不生效时写文件，属主进程在等 turn 时每秒轮询到后
|
||||
用自己的 live 连接发中断（run_turn 的 interrupted-requested 分支）。"""
|
||||
if not thread_id or not turn_id:
|
||||
return None
|
||||
return self.request("turn/interrupt",
|
||||
@@ -1442,21 +1589,36 @@ def dispatch(server, state, name, prompt, repo, model, effort, sandbox,
|
||||
|
||||
|
||||
def run_task_turn(server, state, task, prompt, model, effort,
|
||||
sandbox=None, timeout=None):
|
||||
sandbox=None, timeout=None, interrupt_file=None):
|
||||
# task 自带 sandbox 时优先用它（只读评审任务不能被全局默认覆盖），
|
||||
# 否则用调用方显式传的 sandbox；两者都没有时不传 sandboxPolicy，
|
||||
# 保留 thread 原有设置（旧任务没有 sandbox 记录时尤其重要）。
|
||||
# timeout 为 None 时不传给 server（沿用 server 默认 600），
|
||||
# 这样旧 FakeServer（不接受 timeout 参数）不受影响；显式
|
||||
# 传了才透传，0 由 server 侧解释为不限时。
|
||||
# 传了才透传，0 由 server 侧解释为不限时。interrupt_file 同理：
|
||||
# 不传时不进 server 参数，旧 FakeServer 不受影响；传了时
|
||||
# 属主在等 turn 时每秒轮询它（见 _interrupt_request_path）。
|
||||
use_sandbox = task.get("sandbox") or sandbox
|
||||
started = time.time()
|
||||
turn_kwargs = {}
|
||||
if timeout is not None:
|
||||
turn_kwargs["timeout"] = timeout
|
||||
result = server.run_turn(task["thread_id"], prompt, model=model,
|
||||
effort=effort, sandbox=use_sandbox,
|
||||
**turn_kwargs)
|
||||
if interrupt_file is not None:
|
||||
turn_kwargs["interrupt_file"] = interrupt_file
|
||||
try:
|
||||
result = server.run_turn(task["thread_id"], prompt, model=model,
|
||||
effort=effort, sandbox=use_sandbox,
|
||||
**turn_kwargs)
|
||||
except TypeError as exc:
|
||||
# 兼容旧 server：不接受 interrupt_file 参数时退回不传它
|
||||
# 再跑一次（goal 单测的 FakeServer 就是这款）；其它
|
||||
# TypeError 一律不吞，原样抛出。
|
||||
if "interrupt_file" not in str(exc):
|
||||
raise
|
||||
turn_kwargs.pop("interrupt_file", None)
|
||||
result = server.run_turn(task["thread_id"], prompt, model=model,
|
||||
effort=effort, sandbox=use_sandbox,
|
||||
**turn_kwargs)
|
||||
finished = time.time()
|
||||
record_turn_usage(state, failed=(result["status"] != "completed"))
|
||||
task["turns"].append({"prompt": prompt, "result": result,
|
||||
@@ -1475,7 +1637,8 @@ def run_task_turn(server, state, task, prompt, model, effort,
|
||||
|
||||
|
||||
def run_task_turn_with_retry(server, state, task, prompt, model, effort,
|
||||
sandbox=None, cfg=None, timeout=None):
|
||||
sandbox=None, cfg=None, timeout=None,
|
||||
interrupt_file=None):
|
||||
"""失败自动重试：默认 max_retries=0 时行为不变（只跑一次）；
|
||||
配了重试时，失败后同 prompt 再跑，最多 max_retries 次，
|
||||
每次重试前先查预算，超预算直接停，不烧钱。"""
|
||||
@@ -1497,7 +1660,8 @@ def run_task_turn_with_retry(server, state, task, prompt, model, effort,
|
||||
task["last_text"] = ""
|
||||
break
|
||||
result = run_task_turn(server, state, task, prompt_try,
|
||||
model, effort, sandbox, timeout=timeout)
|
||||
model, effort, sandbox, timeout=timeout,
|
||||
interrupt_file=interrupt_file)
|
||||
# 各次尝试的真实用量都要算（失败尝试花的钱也是钱），
|
||||
# 最后一次的结果里带汇总，不只带最后一次。
|
||||
usage = _add_usage(usage, result.get("usage"))
|
||||
@@ -1566,7 +1730,10 @@ def run_goal_loop(server, state, state_path, task, goal_text, verify,
|
||||
f"验收命令：{verify}\n"
|
||||
f"上一轮验收输出：\n{last_verify}")
|
||||
result = run_task_turn(server, state, task, prompt,
|
||||
model, effort, sandbox, timeout=timeout)
|
||||
model, effort, sandbox, timeout=timeout,
|
||||
interrupt_file=_interrupt_request_path(
|
||||
state_path, task["name"])
|
||||
if task.get("name") else None)
|
||||
goal_usage = _add_usage(goal_usage, result.get("usage"))
|
||||
last_turn_status = result["status"]
|
||||
last_turn_text = result.get("text", "") or ""
|
||||
@@ -1664,6 +1831,11 @@ def main():
|
||||
help="单轮等待上限（秒）；0 表示不限时、对齐 Codex App "
|
||||
"等 turn/completed 的语义；不传时用配置 "
|
||||
"turn_timeout，未配则默认 600")
|
||||
ap.add_argument("--progress-interval", type=float, default=None,
|
||||
help="长等待流式进度间隔（秒）；每满一个间隔往 stderr "
|
||||
"打一行 turn progress 并往链路文件记一条 "
|
||||
"kind=progress；0 表示关闭进度输出；不传时用配置 "
|
||||
"progress_interval，未配则默认 30")
|
||||
sub = ap.add_subparsers(dest="cmd", required=True)
|
||||
|
||||
p = sub.add_parser("init")
|
||||
@@ -1782,6 +1954,12 @@ def main():
|
||||
args.turn_timeout = cfg.get("turn_timeout")
|
||||
if args.turn_timeout is None:
|
||||
args.turn_timeout = 600
|
||||
# progress-interval 解析：同款 CLI > 配置 > 默认 30；0 保持为 0
|
||||
#（关闭进度输出），不能用 or 串联。
|
||||
if args.progress_interval is None:
|
||||
args.progress_interval = cfg.get("progress_interval")
|
||||
if args.progress_interval is None:
|
||||
args.progress_interval = 30
|
||||
args.codex_home = args.codex_home or _resolve_path(cfg_dir, cfg.get("codex_home"))
|
||||
args.codex_bin = args.codex_bin or _resolve_path(cfg_dir, cfg.get("codex_bin"))
|
||||
args.state = args.state or _resolve_path(cfg_dir, cfg.get("state")) or DEFAULT_STATE
|
||||
@@ -2010,7 +2188,8 @@ def main():
|
||||
print("host daemon stopped", flush=True)
|
||||
return
|
||||
srv = Server(codex_bin=args.codex_bin, codex_home=args.codex_home,
|
||||
approval=args.approval)
|
||||
approval=args.approval,
|
||||
progress_interval=args.progress_interval)
|
||||
# 实时链路：只给跑 turn 的命令设链路文件（dispatch/run/goal/
|
||||
# delegate）；只读命令不设。名字映射与 worker 派发一致：
|
||||
# dispatch/goal/delegate 用 --name，run 用 --task。
|
||||
@@ -2042,7 +2221,8 @@ def main():
|
||||
result = run_task_turn_with_retry(srv, state, task, args.task,
|
||||
args.model, args.effort,
|
||||
args.sandbox, cfg=cfg,
|
||||
timeout=args.turn_timeout)
|
||||
timeout=args.turn_timeout,
|
||||
interrupt_file=_interrupt_request_path(args.state, args.name))
|
||||
if result["status"] != "completed":
|
||||
notify_failure(args.state, cfg, args.name, result)
|
||||
save_state(args.state, state)
|
||||
@@ -2104,7 +2284,8 @@ def main():
|
||||
srv.resume_thread(t["thread_id"])
|
||||
result = run_task_turn(srv, state, t, build_fix_prompt(t),
|
||||
args.model, args.effort, args.sandbox,
|
||||
timeout=args.turn_timeout)
|
||||
timeout=args.turn_timeout,
|
||||
interrupt_file=_interrupt_request_path(args.state, args.task))
|
||||
if result["status"] == "completed":
|
||||
for c in oc:
|
||||
c["status"] = "resolved"
|
||||
@@ -2189,7 +2370,8 @@ def main():
|
||||
result = run_task_turn_with_retry(srv, state, t, args.prompt,
|
||||
args.model, args.effort,
|
||||
args.sandbox, cfg=cfg,
|
||||
timeout=args.turn_timeout)
|
||||
timeout=args.turn_timeout,
|
||||
interrupt_file=_interrupt_request_path(args.state, args.task))
|
||||
if result["status"] != "completed":
|
||||
notify_failure(args.state, cfg, args.task, result)
|
||||
save_state(args.state, state)
|
||||
@@ -2206,12 +2388,57 @@ def main():
|
||||
else:
|
||||
srv.resume_thread(t["thread_id"])
|
||||
resp = srv.interrupt_turn(t["thread_id"], turn_id)
|
||||
# 只有服务端接受中断才改状态；它说“没有活动回合”
|
||||
# 时说明该轮已结束，硬标 interrupted 会掩盖真实终态。
|
||||
# 直接中断生效（快路径）时沿用老行为：只有服务端
|
||||
# 接受中断才改状态；它说“没有活动回合”时说明该轮
|
||||
# 已结束，硬标 interrupted 会掩盖真实终态。
|
||||
if resp and "error" not in resp:
|
||||
t["status"] = "interrupted"
|
||||
save_state(args.state, state)
|
||||
print(f"interrupt {args.task}: turn={turn_id} resp={resp}")
|
||||
req = _interrupt_request_path(args.state, args.task)
|
||||
try:
|
||||
if os.path.exists(req):
|
||||
os.remove(req)
|
||||
except Exception:
|
||||
pass
|
||||
print(f"interrupt {args.task}: turn={turn_id} resp={resp}")
|
||||
else:
|
||||
# 直接中断不生效（典型是另一进程在等这个 turn，
|
||||
# 本进程的 server 并不拥有它）：写中断请求文件，
|
||||
# 属主进程在等 turn 时每秒轮询到后用自己的
|
||||
# live 连接发 turn/interrupt 并记 interrupted。
|
||||
# 写完后最多等 10 秒看属主是否已响应，超时也
|
||||
# 不删文件——属主下次轮询时仍会处理，调用方
|
||||
# 也可 Ctrl-C 属主进程 + recover 兜底。
|
||||
req = _interrupt_request_path(args.state, args.task)
|
||||
try:
|
||||
os.makedirs(os.path.dirname(req), exist_ok=True)
|
||||
with open(req, "w", encoding="utf-8") as f:
|
||||
json.dump({"task": args.task,
|
||||
"thread_id": t["thread_id"],
|
||||
"turn_id": turn_id,
|
||||
"requested_at": time.time()},
|
||||
f, ensure_ascii=False)
|
||||
except Exception as exc:
|
||||
print(f"interrupt {args.task}: 写中断请求失败：{exc}")
|
||||
return
|
||||
print(f"interrupt {args.task}: 直接中断未生效 "
|
||||
f"(resp={resp})，已写中断请求 {req}，"
|
||||
f"等待属主进程处理")
|
||||
for _ in range(10):
|
||||
time.sleep(1)
|
||||
try:
|
||||
cur = json.load(open(args.state,
|
||||
encoding="utf-8"))
|
||||
if (cur.get("tasks", {}).get(args.task, {})
|
||||
.get("status") == "interrupted"):
|
||||
print(f"interrupt {args.task}: 属主已响应，"
|
||||
f"任务已记 interrupted")
|
||||
break
|
||||
except Exception:
|
||||
pass
|
||||
else:
|
||||
print(f"interrupt {args.task}: 属主 10 秒内未响应，"
|
||||
f"请求文件仍在 {req}，属主下次轮询时会处理")
|
||||
elif args.cmd == "delegate":
|
||||
parent = state["tasks"][args.parent_task]
|
||||
# 子 thread：cwd 与父任务相同 worktree 时只读更安全，
|
||||
@@ -2228,7 +2455,8 @@ def main():
|
||||
save_state(args.state, state)
|
||||
result = run_task_turn(srv, state, sub, args.task,
|
||||
args.model, args.effort, args.sandbox,
|
||||
timeout=args.turn_timeout)
|
||||
timeout=args.turn_timeout,
|
||||
interrupt_file=_interrupt_request_path(args.state, args.name))
|
||||
save_state(args.state, state)
|
||||
print(json.dumps({"sub_task": args.name, "status": sub["status"],
|
||||
"text": result.get("text", "")},
|
||||
@@ -2239,7 +2467,8 @@ def main():
|
||||
f"子任务 {args.name} 的结论如下，请汇总到主任务结论里，"
|
||||
f"不要重复做子任务的工作：\n{result.get('text', '')}",
|
||||
args.model, args.effort, args.sandbox,
|
||||
timeout=args.turn_timeout)
|
||||
timeout=args.turn_timeout,
|
||||
interrupt_file=_interrupt_request_path(args.state, args.parent_task))
|
||||
save_state(args.state, state)
|
||||
print(f"reported to parent: {rep.get('text', '')[:1000]}")
|
||||
elif args.cmd == "goal":
|
||||
@@ -2361,7 +2590,8 @@ def main():
|
||||
return job["name"], run_task_turn_with_retry(
|
||||
srv, state, t, job["task"], args.model,
|
||||
args.effort, args.sandbox, cfg=cfg,
|
||||
timeout=args.turn_timeout)
|
||||
timeout=args.turn_timeout,
|
||||
interrupt_file=_interrupt_request_path(args.state, job["name"]))
|
||||
with ThreadPoolExecutor(max_workers=len(ready)) as ex:
|
||||
for name, result in ex.map(
|
||||
_run, [tasks[j["name"]] for j in ready]):
|
||||
@@ -2397,7 +2627,8 @@ def main():
|
||||
save_state(args.state, state)
|
||||
result = run_task_turn(srv, state, task, auto["task"],
|
||||
args.model, args.effort, args.sandbox,
|
||||
timeout=args.turn_timeout)
|
||||
timeout=args.turn_timeout,
|
||||
interrupt_file=_interrupt_request_path(args.state, name))
|
||||
auto["last_run"] = now
|
||||
if result["status"] != "completed":
|
||||
notify_failure(args.state, cfg, name, result)
|
||||
|
||||
Reference in new issue
Block a user