host
ae3aff8602
1. host goal 循环连续失败护栏：3 轮连续失败即中断出口， 任务记 interrupted、可断点续跑，不再无限空转 （2026-10-29 实弹：断点后 274 轮全失败）。 2. worker 退出杀残留路径补审计 worker_shutdown_interrupted， detail 为任务 id，与 sweep_interrupted 同口径。 3. 配额中断自动恢复：每轮现查配额，仍超限/查不到时留 在 interrupted 不重排队，下轮再查；现查为 None 时 （新月份/额度上调/无价上限重置）才 CAS 回 queued。 单测：配额仍超限不动、配额恢复续跑、连续 3 轮失败停转。
3291 lines
158 KiB
Python
3291 lines
158 KiB
Python
#!/usr/bin/env python3
|
||
"""
|
||
宿主编排层（无 GUI）：对齐 ChatGPT / Codex 桌面版宿主该干的事，
|
||
而不是再写一个 agent 循环。
|
||
|
||
ChatGPT / Codex 桌面版作为宿主，核心就这几件（见 README 的对齐表）：
|
||
1. 线程生命周期：建、续跑、fork、归档、删除、列表、持久化
|
||
2. 并行派发：多 thread 同时跑，每任务一个 git worktree 隔离
|
||
3. 子代理委派：主 thread 派子 thread，子 thread 结果汇总回主 thread
|
||
（模型内子代理是另一层，见 examples/subagent-prompt.txt；
|
||
这里是宿主级委派，用 parentTurnId / rootTurnId 把委派链串起来）
|
||
4. 长任务 / 目标循环：同一 thread 多回合，直到验收命令通过或到上限
|
||
5. 评审队列：diff + 状态 + 合并 / 丢弃，人工在合并前把关
|
||
6. 自动化：按间隔到期的任务自动派发，结果进评审队列（无推送，
|
||
这是云端 ChatGPT 才有的部分，见 README 未对齐表）
|
||
7. 边界：宿主定 cwd、沙箱、审批、模型、MCP/skills，agent 只在边界内跑
|
||
|
||
实现方式：一个 app-server 进程托管多个 thread（官方 thread manager
|
||
就是这么设计的），本文件用后台读线程按 id / threadId 分发响应和
|
||
turn/completed 通知，所以多个 thread 的 turn 可以并发跑。
|
||
线程状态本身持久化在 CODEX_HOME，host-state.json 只存宿主自己的
|
||
任务登记（task -> thread/worktree/状态/diff），不是 agent 状态的
|
||
第二份拷贝。
|
||
|
||
用法示例：
|
||
python3 host.py dispatch --repo work/host-repo --task "给 sample.py 加 add2 函数" --name add2
|
||
python3 host.py list
|
||
python3 host.py status add2
|
||
python3 host.py review add2
|
||
python3 host.py merge add2 # 评审通过后才合并，冲突会 abort
|
||
python3 host.py discard add2 # 评审不通过，删 worktree 和分支
|
||
python3 host.py run --task add2 --prompt "再补一个测试"
|
||
python3 host.py fork --task add2 --name add2-v2
|
||
python3 host.py delegate --parent-task add2 --task "只读检查 add2 有没有边界问题" --name add2-review
|
||
python3 host.py goal --repo work/host-repo --task "创建 result.txt，内容只有一行 goal-ok" \
|
||
--verify "test -f result.txt" --name goal1
|
||
python3 host.py parallel --jobs examples/host-jobs.json
|
||
python3 host.py automations --add --name daily-check --repo work/host-repo \
|
||
--task "只读检查 sample.py 有没有明显问题" --interval 86400
|
||
python3 host.py run-due
|
||
|
||
默认 CODEX_HOME 是 ~/.codex；测试时用 --codex-home 指向隔离目录。
|
||
默认用 PATH 里的 codex；本机 VM 需用配对二进制时设
|
||
CODEX_BIN=.../bin/codex（start.sh 已自动优先用 bin/codex）。
|
||
"""
|
||
import argparse
|
||
import json
|
||
import os
|
||
import queue
|
||
import re
|
||
import shlex
|
||
import subprocess
|
||
import sys
|
||
import threading
|
||
import time
|
||
import uuid
|
||
from concurrent.futures import ThreadPoolExecutor
|
||
|
||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||
DEFAULT_STATE = os.path.join(HERE, "host-state.json")
|
||
|
||
|
||
def _sum_turn_usage(events, thread_id, turn_id):
|
||
"""从事件窗口里汇总一回合的上游真实用量。
|
||
|
||
app-server 在每个上游补全结束时发 thread/tokenUsage/updated，
|
||
params.tokenUsage.last 是该次补全的用量（2026-10-26 中继实测：
|
||
同一 turn 的多次 last 之和等于最后一次的 total，input/output/
|
||
cached/reasoning 四路都对得上）。按 threadId + turnId 过滤后
|
||
累加 last，得到本回合的真实用量；无匹配通知时返回 None（老
|
||
版本 server 或回合未产生补全时调用方回落估算，绝不把 0 当成
|
||
真实用量）。注意 totalTokens = inputTokens + outputTokens，
|
||
reasoning 包含在 output 里，累加时不另算。
|
||
"""
|
||
fields = (("input_tokens", "inputTokens"),
|
||
("cached_input_tokens", "cachedInputTokens"),
|
||
("cache_write_input_tokens", "cacheWriteInputTokens"),
|
||
("output_tokens", "outputTokens"),
|
||
("reasoning_output_tokens", "reasoningOutputTokens"),
|
||
("total_tokens", "totalTokens"))
|
||
total = None
|
||
for ev in events or []:
|
||
if ev.get("method") != "thread/tokenUsage/updated":
|
||
continue
|
||
p = ev.get("params") or {}
|
||
if p.get("threadId") != thread_id:
|
||
continue
|
||
if turn_id is not None and p.get("turnId") != turn_id:
|
||
continue
|
||
last = (p.get("tokenUsage") or {}).get("last") or {}
|
||
if total is None:
|
||
total = {k: 0 for k, _ in fields}
|
||
total["completions"] = 0
|
||
for key, wire in fields:
|
||
total[key] += int(last.get(wire) or 0)
|
||
total["completions"] += 1
|
||
return total
|
||
|
||
|
||
def _partial_text_from_events(events):
|
||
"""超时回合的 best-effort 部分文本：从事件窗口里已完成的
|
||
agentMessage 收集文本（不含流式 delta，避免重复拼接）。
|
||
找不到时返回空串，调用方不得把部分文本当成最终结论。"""
|
||
parts = []
|
||
for ev in events or []:
|
||
p = ev.get("params") or {}
|
||
item = p.get("item") or {}
|
||
if item.get("type") == "agentMessage" and item.get("text"):
|
||
text = item["text"]
|
||
if not parts or parts[-1] != text:
|
||
parts.append(text)
|
||
return "\n".join(parts)
|
||
|
||
|
||
def _interrupt_request_path(state_path, task_name):
|
||
"""跨进程中断请求文件路径：<state_dir>/interrupt-requests/<task>.json。
|
||
|
||
属主进程在等 turn 时每秒轮询这个文件，另一个进程的
|
||
`host.py interrupt --task` 直接协议中断不生效时就写这个文件，
|
||
属主用自己的 live 连接发 turn/interrupt——只有属主的连接
|
||
一定拥有这个 turn，这是跨进程中断可靠的原因。文件本身不
|
||
由 run_task_turn 创建，只由 interrupt 命令写、属主读到后删。
|
||
"""
|
||
safe = re.sub(r"[^A-Za-z0-9_.-]", "-", str(task_name))
|
||
return os.path.join(
|
||
os.path.dirname(os.path.abspath(state_path)),
|
||
"interrupt-requests", safe + ".json")
|
||
|
||
|
||
def _wait_turn(q, timeout, progress_interval=30, on_progress=None,
|
||
interrupt_check=None):
|
||
"""等 turn/completed 的轮询等待（对齐 Codex App 的长等待语义）。
|
||
|
||
原来是一次性 q.get(timeout) / q.get()，长等待全程静默、另一
|
||
进程无法插手；现在按 1 秒切片轮询，每片做两件事：
|
||
1) interrupt_check() 为真时立刻返回 ("interrupted-requested", None)，
|
||
由调用方用自己的 live 连接发 turn/interrupt 再记 interrupted；
|
||
2) progress_interval > 0 且距上次进度已满一个间隔时调
|
||
on_progress(elapsed_seconds)，让调用方打流式进度。
|
||
返回：("completed", done) / ("timeout", None) /
|
||
("interrupted-requested", None)。timeout 为 None 或 <=0 时
|
||
不限时，只等 done 或中断请求；正数时到时返回 timeout。
|
||
"""
|
||
start = time.monotonic()
|
||
next_progress = progress_interval if progress_interval and \
|
||
progress_interval > 0 else None
|
||
while True:
|
||
elapsed = time.monotonic() - start
|
||
if timeout and timeout > 0:
|
||
remaining = timeout - elapsed
|
||
if remaining <= 0:
|
||
return "timeout", None
|
||
slice_s = min(1.0, remaining)
|
||
else:
|
||
slice_s = 1.0
|
||
# 切片还要受下一个进度点约束，否则一个长切片会把
|
||
# 中间的进度检查全吞掉（timeout=0.35、间隔=0.1 时只会
|
||
# 打 1 次进度，而不是 3 次；单测抓住了这个 bug）。
|
||
if next_progress is not None:
|
||
slice_s = min(slice_s, max(0.01, next_progress - elapsed))
|
||
try:
|
||
done = q.get(timeout=slice_s)
|
||
return "completed", done
|
||
except queue.Empty:
|
||
pass
|
||
elapsed = time.monotonic() - start
|
||
if interrupt_check is not None:
|
||
try:
|
||
if interrupt_check():
|
||
return "interrupted-requested", None
|
||
except Exception:
|
||
pass
|
||
if next_progress is not None and elapsed >= next_progress:
|
||
if on_progress is not None:
|
||
try:
|
||
on_progress(elapsed)
|
||
except Exception:
|
||
pass
|
||
next_progress += progress_interval
|
||
if timeout and timeout > 0 and elapsed >= timeout:
|
||
return "timeout", None
|
||
|
||
|
||
def _add_usage(a, b):
|
||
"""两份用量相加（任一为空时返回另一份；都为空返回 None）。"""
|
||
if not a:
|
||
return dict(b) if b else None
|
||
if not b:
|
||
return dict(a)
|
||
out = dict(a)
|
||
for k, v in b.items():
|
||
if isinstance(v, int):
|
||
out[k] = out.get(k, 0) + v
|
||
return out
|
||
|
||
|
||
def default_codex_bin():
|
||
if os.environ.get("CODEX_BIN"):
|
||
return os.environ["CODEX_BIN"]
|
||
paired = os.path.join(HERE, "bin", "codex")
|
||
if os.path.exists(paired):
|
||
return paired
|
||
return "codex"
|
||
|
||
|
||
# 实时链路：单条通知里 item 的上限行数（防失控回合把文件写爆）。
|
||
TRACE_MAX_LINES = 5000
|
||
|
||
# agent shell 绝不能看到的变量（codex shell_environment_policy.exclude）。
|
||
# 实弹证实：不设此策略时 agent shell 直接继承 server 进程 env，
|
||
# XCM_API_KEY 在 local 路可被模型拿到；server 自己的中转调用走
|
||
# 进程 env（env_key），不受 shell 策略影响，所以只排 shell 不影响
|
||
# 模型调用。inherit 保持默认 all（PATH、代理变量等工作流变量不
|
||
# 动）；gh 用文件登录不用 env，故排 *_TOKEN 不破坏 git 流程
|
||
# （单测验证：gh 登录路径不读 token 环境变量，见 test_shell_env）。
|
||
# P0 安全批后新增 *_TOKEN、*_AUTH*、*_CREDENTIAL*、*API_KEY*
|
||
# 四条模式，兜底将来新增的同类变量。
|
||
# 残留风险（必须如实写清，不谎称修完）：这只是 shell 过滤，
|
||
# 不是安全边界——agent 与 server 进程同 UID 时，模型仍可读
|
||
# server 进程的 /proc/<pid>/environ 或其它未过滤变量；真正
|
||
# 隔离需要 UID 分离或 hidepid 的部署变更，不在本批代码里，
|
||
# 部署清单里必须单独列为后续项。
|
||
SHELL_ENV_EXCLUDE = [
|
||
"XCM_API_KEY",
|
||
"CLOUD_DB_PASSWORD",
|
||
"GITHUB_OAUTH_CLIENT_SECRET",
|
||
"GITHUB_WEBHOOK_SECRET",
|
||
"*_KEY",
|
||
"*_SECRET",
|
||
"*_PASSWORD",
|
||
"*_TOKEN",
|
||
"*_AUTH*",
|
||
"*_CREDENTIAL*",
|
||
"*API_KEY*",
|
||
]
|
||
|
||
|
||
def _summarize_trace_item(method, params):
|
||
"""app-server 通知 → 链路条目列表（实时链路用）。
|
||
|
||
只取过程事实，不取大文本：命令截 500 字、消息截 300 字、
|
||
思考摘要截 200 字；fileChange 一项一变化拆多条。
|
||
返回 [] 表示该通知不进链路（userMessage 等）。
|
||
ts 取通知的 completedAtMs/startedAtMs（毫秒 epoch），
|
||
没有时为 None，服务端展示时按无时间处理。
|
||
thread 取通知的 threadId（模型内子代理与主任务同文件时，
|
||
服务端凭它把条目归属到不同代理；没有时为 None，归主任务）。
|
||
"""
|
||
item = (params or {}).get("item")
|
||
if not isinstance(item, dict):
|
||
return []
|
||
item_type = item.get("type")
|
||
item_id = item.get("id") or ""
|
||
ts = params.get("completedAtMs") or params.get("startedAtMs")
|
||
thread = params.get("threadId")
|
||
if not thread and isinstance(params.get("thread"), dict):
|
||
thread = params["thread"].get("id")
|
||
if item_type == "commandExecution":
|
||
return [{"kind": "exec", "id": item_id, "ts": ts, "thread": thread,
|
||
"command": (item.get("command") or "")[:500],
|
||
"cwd": item.get("cwd") or "",
|
||
"exitCode": item.get("exitCode"),
|
||
"durationMs": item.get("durationMs")}]
|
||
if item_type == "fileChange":
|
||
entries = []
|
||
for ch in item.get("changes") or []:
|
||
if not isinstance(ch, dict):
|
||
continue
|
||
path = ch.get("path") or ""
|
||
entries.append({"kind": "file",
|
||
"id": item_id + ":" + path, "ts": ts,
|
||
"thread": thread,
|
||
"path": path, "change": ch.get("kind") or "",
|
||
"status": item.get("status") or ""})
|
||
return entries
|
||
if item_type == "agentMessage":
|
||
text = item.get("text") or ""
|
||
if not text:
|
||
return []
|
||
return [{"kind": "message", "id": item_id, "ts": ts, "thread": thread,
|
||
"phase": item.get("phase") or "", "text": text[:300]}]
|
||
if item_type == "reasoning":
|
||
summary = item.get("summary") or item.get("content") or []
|
||
text = " ".join(s for s in summary if isinstance(s, str)).strip()
|
||
if not text:
|
||
return []
|
||
return [{"kind": "reasoning", "id": item_id, "ts": ts, "thread": thread,
|
||
"summary": text[:200]}]
|
||
if item_type == "mcpToolCall":
|
||
return [{"kind": "tool", "id": item_id, "ts": ts, "thread": thread,
|
||
"server": item.get("server") or "",
|
||
"tool": item.get("tool") or "",
|
||
"status": item.get("status") or ""}]
|
||
if item_type == "functionCallOutput":
|
||
return [{"kind": "call", "id": item_id, "ts": ts, "thread": thread,
|
||
"name": item.get("name") or ""}]
|
||
if item_type == "plan":
|
||
text = item.get("text") or ""
|
||
if not text:
|
||
return []
|
||
return [{"kind": "plan", "id": item_id, "ts": ts, "thread": thread,
|
||
"text": text[:300]}]
|
||
return []
|
||
|
||
|
||
def _trace_file_path(state_path, task_name):
|
||
"""链路文件路径：state 文件同目录下的 host-traces/<清洗后名字>.jsonl。
|
||
|
||
与 Server.set_trace_file、cloud 的 _trace_file_name 同款清洗，
|
||
三边算出同一名字，delegate 的父链路标记才能落进父任务文件。
|
||
"""
|
||
safe = re.sub(r"[^A-Za-z0-9_.-]", "-", str(task_name))
|
||
return os.path.join(
|
||
os.path.dirname(os.path.abspath(state_path)),
|
||
"host-traces", safe + ".jsonl")
|
||
|
||
|
||
def _append_trace_marker(state_path, task_name, entry):
|
||
"""往另一任务的链路文件追加一条标记（delegate 向父链路报到用）。
|
||
|
||
尽力而为：目录建不出、写失败都不影响回合本身。父链路的
|
||
行数上限由属主进程自己的计数器管，这里不重复计数（标记
|
||
一轮至多两条，超限风险可忽略）。
|
||
"""
|
||
try:
|
||
path = _trace_file_path(state_path, task_name)
|
||
os.makedirs(os.path.dirname(path), exist_ok=True)
|
||
with open(path, "a", encoding="utf-8") as f:
|
||
f.write(json.dumps(entry, ensure_ascii=False) + "\n")
|
||
except Exception:
|
||
pass
|
||
|
||
|
||
def _server_command(codex_bin, provider=None):
|
||
"""app-server 启动命令：-c 必须放在子命令之前。
|
||
|
||
shell_environment_policy.exclude 只过滤 agent shell 看到的
|
||
变量，不影响 server 进程自己的中转调用（env_key 直读进程 env）。
|
||
provider="sub2api" 时再叠加 -c 覆盖：model_provider 与
|
||
provider 表都按任务进程生效，不改任何租户的 config.toml
|
||
（多网关任务同项目并发时互不干扰）。网关地址取环境变量
|
||
SUB2API_BASE_URL（origin，不带 /v1）；未配则 loud fail，
|
||
绝不静默跑回默认网关。
|
||
"""
|
||
cmd = [
|
||
codex_bin,
|
||
"-c",
|
||
"shell_environment_policy.exclude=" + json.dumps(SHELL_ENV_EXCLUDE),
|
||
]
|
||
if provider is not None:
|
||
if provider != "sub2api":
|
||
raise ValueError("未知模型 provider: %r" % provider)
|
||
origin = os.environ.get("SUB2API_BASE_URL", "").strip().rstrip("/")
|
||
if not origin:
|
||
raise RuntimeError(
|
||
"provider=sub2api 但 SUB2API_BASE_URL 未配置")
|
||
cmd += [
|
||
"-c", 'model_provider="sub2api"',
|
||
"-c", 'model_providers.sub2api.name="Sub2API"',
|
||
"-c", 'model_providers.sub2api.base_url="%s/v1"' % origin,
|
||
"-c", 'model_providers.sub2api.env_key="SUB2API_API_KEY"',
|
||
"-c", 'model_providers.sub2api.wire_api="responses"',
|
||
]
|
||
cmd.append("app-server")
|
||
return cmd
|
||
|
||
|
||
class Server:
|
||
"""单个 app-server 进程，多 thread 并发。"""
|
||
|
||
def __init__(self, codex_bin=None, codex_home=None, approval="deny",
|
||
progress_interval=30, model_provider=None):
|
||
self.codex_bin = codex_bin or default_codex_bin()
|
||
self.codex_home = codex_home
|
||
self.approval = approval
|
||
self.model_provider = model_provider
|
||
# 流式进度间隔（秒）：run_turn 等待时每满一个间隔打一次
|
||
# 进度；0 表示关闭进度输出，但中断轮询仍每秒进行。
|
||
self.progress_interval = progress_interval
|
||
self.send_lock = threading.Lock()
|
||
self.next_id = 1
|
||
self.pending = {}
|
||
self.pending_lock = threading.Lock()
|
||
self.turn_queues = {}
|
||
self.events = []
|
||
self.events_lock = threading.Lock()
|
||
self.restart_count = 0
|
||
# 实时链路文件：set_trace_file 未调时为 None，回合本身
|
||
# 不受影响（写链路失败也绝不打断 _read_loop）。
|
||
self.trace_path = None
|
||
self._trace_lines = 0
|
||
self._start()
|
||
|
||
def set_trace_file(self, state_path, task_name):
|
||
"""链路文件定到 <state_dir>/host-traces/<task_name>.jsonl。
|
||
|
||
state_dir 与 state 文件同目录：本地 worker 是
|
||
/opt/codex-app-server/work，容器内是 /app/work（经挂载
|
||
落到宿主同一目录），服务端按同一规则读。目录建不出
|
||
时 trace_path 保持 None，不影响回合。
|
||
"""
|
||
try:
|
||
self.trace_path = _trace_file_path(state_path, task_name)
|
||
os.makedirs(os.path.dirname(self.trace_path), exist_ok=True)
|
||
self._trace_lines = 0
|
||
except Exception:
|
||
self.trace_path = None
|
||
|
||
def _append_trace(self, msg):
|
||
"""通知 → 摘要条目 → 追加写链路文件（每事件开合一次文件，
|
||
进程被杀也不丢已写行；行数到 TRACE_MAX_LINES 即停）。"""
|
||
if self.trace_path is None or \
|
||
self._trace_lines >= TRACE_MAX_LINES:
|
||
return
|
||
entries = _summarize_trace_item(msg.get("method"),
|
||
msg.get("params") or {})
|
||
if not entries:
|
||
return
|
||
with open(self.trace_path, "a", encoding="utf-8") as f:
|
||
for entry in entries:
|
||
if self._trace_lines >= TRACE_MAX_LINES:
|
||
break
|
||
f.write(json.dumps(entry, ensure_ascii=False) + "\n")
|
||
self._trace_lines += 1
|
||
|
||
def _start(self):
|
||
env = dict(os.environ)
|
||
if self.codex_home:
|
||
os.makedirs(self.codex_home, exist_ok=True)
|
||
env["CODEX_HOME"] = self.codex_home
|
||
self.proc = subprocess.Popen(
|
||
_server_command(self.codex_bin,
|
||
provider=self.model_provider),
|
||
stdin=subprocess.PIPE, stdout=subprocess.PIPE,
|
||
stderr=subprocess.DEVNULL, env=env,
|
||
)
|
||
self.reader = threading.Thread(target=self._read_loop, daemon=True)
|
||
self.reader.start()
|
||
|
||
def is_alive(self):
|
||
return self.proc is not None and self.proc.poll() is None
|
||
|
||
def ensure_alive(self):
|
||
"""server 崩了就重拉一次，thread 存在 CODEX_HOME 里，
|
||
重拉后靠 resume_thread 恢复，不丢 thread。
|
||
返回 True 表示 alive（原来 alive 或重拉成功）。"""
|
||
if self.is_alive():
|
||
return True
|
||
try:
|
||
self._start()
|
||
self.initialize()
|
||
self.restart_count += 1
|
||
return True
|
||
except Exception:
|
||
return False
|
||
|
||
def _answer_server_request(self, msg):
|
||
# MCP 工具审批不是通用 approval：服务端发的是
|
||
# mcpServer/elicitation/request（mode=form，requestedSchema），
|
||
# 按 MCP elicitation 协议回 action=accept/decline + content，
|
||
# 回通用 {"decision": ...} 会被当成拒绝（2026-10-25 实测）。
|
||
if msg.get("method") == "mcpServer/elicitation/request":
|
||
if self.approval == "auto":
|
||
result = {"action": "accept", "content": {}}
|
||
else:
|
||
result = {"action": "decline"}
|
||
self._send({"jsonrpc": "2.0", "id": msg["id"], "result": result})
|
||
return
|
||
decision = "approve" if self.approval == "auto" else "deny"
|
||
self._send({"jsonrpc": "2.0", "id": msg["id"],
|
||
"result": {"decision": decision}})
|
||
|
||
def _read_loop(self):
|
||
try:
|
||
for raw in self.proc.stdout:
|
||
if not raw.strip():
|
||
continue
|
||
try:
|
||
msg = json.loads(raw)
|
||
except Exception:
|
||
continue
|
||
if "method" in msg and "id" in msg:
|
||
self._answer_server_request(msg)
|
||
continue
|
||
if "id" in msg:
|
||
with self.pending_lock:
|
||
q = self.pending.get(msg["id"])
|
||
if q is not None:
|
||
q.put(msg)
|
||
continue
|
||
# 通知
|
||
params = msg.get("params") or {}
|
||
thread_id = params.get("threadId")
|
||
if not thread_id and isinstance(params.get("thread"), dict):
|
||
thread_id = params["thread"].get("id")
|
||
with self.events_lock:
|
||
self.events.append(msg)
|
||
if self.trace_path is not None:
|
||
try:
|
||
self._append_trace(msg)
|
||
except Exception:
|
||
pass
|
||
if msg.get("method") == "turn/completed" and thread_id:
|
||
q = self.turn_queues.get(thread_id)
|
||
if q is not None:
|
||
q.put(msg)
|
||
except Exception:
|
||
pass
|
||
|
||
def _send(self, obj):
|
||
with self.send_lock:
|
||
self.proc.stdin.write((json.dumps(obj) + "\n").encode())
|
||
self.proc.stdin.flush()
|
||
|
||
def request(self, method, params=None, timeout=120):
|
||
# BrokenPipe / 进程已死时重拉一次再重试（只重试一次，
|
||
# turn/start 这类有副作用的调用由上层 resume 后再跑，
|
||
# 这里重试的是 initialize/list/resume 等幂等请求为主；
|
||
# 即使是 turn/start，重试也只发生在“还没收到任何响应”
|
||
# 的 BrokenPipe 场景，server 死在 turn 中途时 _read_loop
|
||
# 已退出、q.get 会超时而不是走这条路）。
|
||
for attempt in range(2):
|
||
rid = self.next_id
|
||
self.next_id += 1
|
||
q = queue.Queue()
|
||
with self.pending_lock:
|
||
self.pending[rid] = q
|
||
try:
|
||
self._send({"jsonrpc": "2.0", "id": rid, "method": method,
|
||
"params": params or {}})
|
||
return q.get(timeout=timeout)
|
||
except (BrokenPipeError, OSError):
|
||
with self.pending_lock:
|
||
self.pending.pop(rid, None)
|
||
if attempt == 0 and self.ensure_alive():
|
||
continue
|
||
raise
|
||
finally:
|
||
with self.pending_lock:
|
||
self.pending.pop(rid, None)
|
||
|
||
def notify(self, method, params=None):
|
||
self._send({"jsonrpc": "2.0", "method": method, "params": params or {}})
|
||
|
||
def initialize(self):
|
||
resp = self.request("initialize", {
|
||
"clientInfo": {"name": "codex-host", "version": "0.1"},
|
||
}, timeout=60)
|
||
if not resp or "error" in resp:
|
||
raise RuntimeError(f"initialize 失败: {resp}")
|
||
self.notify("initialized", {})
|
||
return resp["result"]
|
||
|
||
def create_thread(self, cwd, model=None, sandbox="workspace-write",
|
||
approval_policy="never", instructions=None):
|
||
params = {"cwd": os.path.abspath(cwd),
|
||
"approvalPolicy": approval_policy, "sandbox": sandbox}
|
||
if model:
|
||
params["model"] = model
|
||
if instructions:
|
||
params["developerInstructions"] = instructions
|
||
resp = self.request("thread/start", params, timeout=120)
|
||
if not resp or "error" in resp:
|
||
raise RuntimeError(f"thread/start 失败: {resp}")
|
||
return resp["result"]["thread"]["id"]
|
||
|
||
def list_threads(self):
|
||
resp = self.request("thread/list", {}, timeout=60)
|
||
if not resp or "error" in resp:
|
||
raise RuntimeError(f"thread/list 失败: {resp}")
|
||
return resp["result"]
|
||
|
||
def resume_thread(self, thread_id):
|
||
resp = self.request("thread/resume", {"threadId": thread_id}, timeout=120)
|
||
if not resp or "error" in resp:
|
||
raise RuntimeError(f"thread/resume 失败: {resp}")
|
||
return resp["result"]
|
||
|
||
def start_review(self, thread_id, target, timeout=600):
|
||
# 协议 review/start：params 必填 threadId 与 target，
|
||
# target 四种见 protocol v2 的 ReviewTarget（uncommitted
|
||
# Changes / baseBranch / commit / custom）；delivery 缺
|
||
# 省即 inline，不发 detached（已废弃）。失败直接抛错，
|
||
# 调用方不得把 error 响应当成功打印。
|
||
params = {"threadId": thread_id, "target": target}
|
||
resp = self.request("review/start", params, timeout=timeout)
|
||
if not resp or "error" in resp:
|
||
raise RuntimeError(f"review/start 失败: {resp}")
|
||
return resp["result"]
|
||
|
||
def fork_thread(self, thread_id, cwd=None, sandbox=None):
|
||
params = {"threadId": thread_id}
|
||
if cwd:
|
||
params["cwd"] = os.path.abspath(cwd)
|
||
if sandbox:
|
||
params["sandbox"] = sandbox
|
||
resp = self.request("thread/fork", params, timeout=120)
|
||
if not resp or "error" in resp:
|
||
raise RuntimeError(f"thread/fork 失败: {resp}")
|
||
# fork 响应里 thread 字段形态随版本略有差异，尽量都取
|
||
r = resp["result"]
|
||
if isinstance(r, dict):
|
||
t = r.get("thread") or {}
|
||
return t.get("id") or r.get("threadId") or r.get("id")
|
||
return None
|
||
|
||
def archive_thread(self, thread_id):
|
||
resp = self.request("thread/archive", {"threadId": thread_id}, timeout=60)
|
||
if resp and "error" in resp:
|
||
raise RuntimeError(f"thread/archive 失败: {resp}")
|
||
return resp
|
||
|
||
def run_turn(self, thread_id, prompt, model=None, effort=None,
|
||
parent_turn_id=None, root_turn_id=None, timeout=600,
|
||
sandbox=None, progress_interval=None, interrupt_file=None):
|
||
# 开 turn 前先看 server 是否还活着，死了就重拉并 resume
|
||
# 这个 thread（thread 持久化在 CODEX_HOME 里，重拉不丢），
|
||
# 否则 turn/start 会直接 BrokenPipe 失败。
|
||
if not self.is_alive():
|
||
if self.ensure_alive():
|
||
try:
|
||
self.resume_thread(thread_id)
|
||
except Exception:
|
||
pass
|
||
q = queue.Queue()
|
||
self.turn_queues[thread_id] = q
|
||
start_events = len(self.events)
|
||
try:
|
||
params = {"threadId": thread_id,
|
||
"input": [{"type": "text", "text": prompt}]}
|
||
if model:
|
||
params["model"] = model
|
||
if effort:
|
||
params["effort"] = effort
|
||
if sandbox:
|
||
# turn 级 sandboxPolicy 会覆盖 thread 级设置，且对后续
|
||
# turn 也生效。这是修复 fork 换 cwd 后被重置为 read-only
|
||
# 的关键：fork 时传 sandbox 参数服务端不认，turn 里传才认。
|
||
tmap = {"workspace-write": "workspaceWrite",
|
||
"read-only": "readOnly",
|
||
"danger-full-access": "dangerFullAccess"}
|
||
params["sandboxPolicy"] = {"type": tmap.get(sandbox, sandbox)}
|
||
if parent_turn_id:
|
||
params["parentTurnId"] = parent_turn_id
|
||
if root_turn_id:
|
||
params["rootTurnId"] = root_turn_id
|
||
resp = self.request("turn/start", params, timeout=120)
|
||
if not resp or "error" in resp:
|
||
return {"status": "failed", "text": "", "error": resp}
|
||
turn_id = None
|
||
if isinstance(resp.get("result"), dict):
|
||
t = resp["result"].get("turn") or {}
|
||
turn_id = t.get("id")
|
||
try:
|
||
# 对齐 Codex App 的长任务语义：timeout 为 None 或 0
|
||
# 表示不限时、一直等到 turn/completed（由用户中断
|
||
# 或模型自己结束）；正数才按秒计时。默认仍是 600，
|
||
# 由调用方（CLI --turn-timeout / 配置 turn_timeout）
|
||
# 显式改，不改默认就不会悄悄挂起自动化。
|
||
# 等待不再是一次性 q.get()：按 1 秒切片轮询，每满
|
||
# progress_interval 打一次流式进度（stderr，绝不
|
||
# 走 stdout——dispatch/run 的 stdout 是 JSON、goal
|
||
# 的 stdout 要被 worker 解析结论行与用量行，进度
|
||
# 进 stdout 会把这两种解析都打坏；链路文件里也
|
||
# 同步记一条 kind=progress，console 按 fallback
|
||
# 把它当 JSON 行展示），并每秒检查跨进程中断
|
||
# 请求文件（见 _interrupt_request_path）。
|
||
interval = self.progress_interval if \
|
||
progress_interval is None else progress_interval
|
||
|
||
def _on_progress(elapsed):
|
||
sys.stderr.write(
|
||
"turn progress: elapsed=%ds\n"
|
||
% int(elapsed))
|
||
sys.stderr.flush()
|
||
if self.trace_path is not None and \
|
||
self._trace_lines < TRACE_MAX_LINES:
|
||
try:
|
||
with self.events_lock:
|
||
n_events = len(self.events)
|
||
with open(self.trace_path, "a",
|
||
encoding="utf-8") as f:
|
||
f.write(json.dumps(
|
||
{"kind": "progress",
|
||
"elapsed_s": int(elapsed),
|
||
"events": n_events},
|
||
ensure_ascii=False) + "\n")
|
||
self._trace_lines += 1
|
||
except Exception:
|
||
pass
|
||
|
||
def _interrupt_check():
|
||
return bool(interrupt_file) and \
|
||
os.path.exists(interrupt_file)
|
||
|
||
wait_status, done = _wait_turn(
|
||
q, timeout, progress_interval=interval,
|
||
on_progress=_on_progress,
|
||
interrupt_check=_interrupt_check)
|
||
if wait_status == "interrupted-requested":
|
||
# 属主用自己的 live 连接发中断：只有这个连接
|
||
# 一定拥有该 turn，另一进程的直接中断未必生效。
|
||
# 中断请求本身已明确，属主不再继续等：即使
|
||
# 服务端回 error 也记 interrupted（任务可续跑，
|
||
# 不是失败），并把请求文件删掉避免重复触发；
|
||
# 服务端侧的停止是 best-effort，stderr 留一行。
|
||
try:
|
||
resp = self.interrupt_turn(thread_id, turn_id)
|
||
if resp and "error" in resp:
|
||
sys.stderr.write(
|
||
"turn interrupt request returned error: "
|
||
"%s\n" % (resp,))
|
||
sys.stderr.flush()
|
||
except Exception as exc:
|
||
sys.stderr.write(
|
||
"turn interrupt request failed: %s\n" % (exc,))
|
||
sys.stderr.flush()
|
||
try:
|
||
if interrupt_file:
|
||
os.remove(interrupt_file)
|
||
except Exception:
|
||
pass
|
||
with self.events_lock:
|
||
evs = list(self.events[start_events:])
|
||
return {"status": "interrupted",
|
||
"text": _partial_text_from_events(evs),
|
||
"turn_id": turn_id,
|
||
"usage": _sum_turn_usage(evs, thread_id, turn_id)}
|
||
if wait_status == "timeout":
|
||
with self.events_lock:
|
||
evs = list(self.events[start_events:])
|
||
return {"status": "timeout",
|
||
"text": _partial_text_from_events(evs),
|
||
"turn_id": turn_id,
|
||
"usage": _sum_turn_usage(evs, thread_id, turn_id)}
|
||
except queue.Empty:
|
||
with self.events_lock:
|
||
evs = list(self.events[start_events:])
|
||
return {"status": "timeout",
|
||
"text": _partial_text_from_events(evs),
|
||
"turn_id": turn_id,
|
||
"usage": _sum_turn_usage(evs, thread_id, turn_id)}
|
||
turn = (done.get("params") or {}).get("turn") or {}
|
||
text = ""
|
||
for item in turn.get("items") or []:
|
||
if item.get("type") == "agentMessage" and \
|
||
item.get("phase") == "final_answer":
|
||
text = item.get("text") or text
|
||
if not text:
|
||
# 兜底：从本回合事件里找 final_answer
|
||
with self.events_lock:
|
||
evs = list(self.events[start_events:])
|
||
for ev in evs:
|
||
p = ev.get("params") or {}
|
||
item = p.get("item") or {}
|
||
if item.get("type") == "agentMessage" and \
|
||
item.get("phase") == "final_answer" and \
|
||
item.get("text"):
|
||
text = item["text"]
|
||
status = turn.get("status", "completed")
|
||
error = turn.get("error")
|
||
if status != "completed" and not error:
|
||
# app-server 的 turn/completed 事件常常只给
|
||
# status=failed，不给失败原因（采样失败的 404
|
||
# 只写在它本地的 thread_history 库里，事件里没有）。
|
||
# 此时 best-effort 从 codex_home 的 thread_history
|
||
# 库里按 thread_id + turn_id 查 error_json，把
|
||
# message 捞回来给调用方（worker 靠它判断是否可
|
||
# fallback）；查不到、库不存在、表结构变了都只当
|
||
# 没有原因，绝不因此把回合本身搞失败。
|
||
error = self._turn_error_from_history(
|
||
thread_id, turn.get("id") or turn_id)
|
||
# 真实用量：本回合窗口内的 tokenUsage/updated 通知累加。
|
||
# 失败/超时回合也一样记（已产生的补全就是已花的钱）。
|
||
with self.events_lock:
|
||
evs = list(self.events[start_events:])
|
||
usage = _sum_turn_usage(
|
||
evs, thread_id, turn.get("id") or turn_id)
|
||
return {"status": status, "text": text,
|
||
"turn_id": turn.get("id") or turn_id,
|
||
"root_turn_id": turn.get("rootTurnId"),
|
||
"error": error, "usage": usage}
|
||
finally:
|
||
self.turn_queues.pop(thread_id, None)
|
||
|
||
def interrupt_turn(self, thread_id, turn_id):
|
||
"""对齐 Codex App 的停止按钮：向 app-server 发 turn/interrupt。
|
||
协议要求 threadId + turnId；turn_id 缺失时无法中断，返回 None。
|
||
注意：直接中断只对本 server 进程正在跑的 turn 可靠，另一进程的
|
||
turn 未必能被本进程的中断请求停下；跨进程的可靠路径是
|
||
中断请求文件（见 _interrupt_request_path）：interrupt 命令
|
||
直接中断不生效时写文件，属主进程在等 turn 时每秒轮询到后
|
||
用自己的 live 连接发中断（run_turn 的 interrupted-requested 分支）。"""
|
||
if not thread_id or not turn_id:
|
||
return None
|
||
return self.request("turn/interrupt",
|
||
{"threadId": thread_id, "turnId": turn_id},
|
||
timeout=120)
|
||
|
||
def _turn_error_from_history(self, thread_id, turn_id):
|
||
"""从 codex thread_history 库里取失败回合的 error message。
|
||
|
||
只读、只在回合失败且事件本身没给原因时调用；任何异常
|
||
都返回 None。库路径：codex_home/thread_history_1.sqlite，
|
||
codex_home 未指定时用 ~/.codex。
|
||
"""
|
||
if not thread_id or not turn_id:
|
||
return None
|
||
try:
|
||
import sqlite3
|
||
home = self.codex_home or os.path.expanduser("~/.codex")
|
||
db_path = os.path.join(home, "thread_history_1.sqlite")
|
||
if not os.path.exists(db_path):
|
||
return None
|
||
conn = sqlite3.connect("file:%s?mode=ro" % db_path, uri=True)
|
||
try:
|
||
row = conn.execute(
|
||
"SELECT error_json FROM thread_turns"
|
||
" WHERE thread_id = ? AND turn_id = ?",
|
||
(thread_id, turn_id)).fetchone()
|
||
finally:
|
||
conn.close()
|
||
if not row or not row[0]:
|
||
return None
|
||
parsed = json.loads(row[0])
|
||
msg = parsed.get("message") if isinstance(parsed, dict) else None
|
||
if not msg:
|
||
return None
|
||
return str(msg)[:1000]
|
||
except Exception:
|
||
return None
|
||
|
||
def close(self):
|
||
try:
|
||
self.proc.stdin.close()
|
||
except Exception:
|
||
pass
|
||
try:
|
||
self.proc.wait(timeout=5)
|
||
except Exception:
|
||
self.proc.kill()
|
||
try:
|
||
self.proc.stdout.close()
|
||
except Exception:
|
||
pass
|
||
|
||
|
||
# ---------------- 宿主状态 / worktree / diff ----------------
|
||
# 生产加固：state 锁。锁只包住“读/写”这两个短临界区，不包住
|
||
# 整个模型 turn（turn 可能跑几分钟，包住它会把别的 host 进程
|
||
# 活活饿死）。保存时先在锁里重读磁盘，再把内存里的任务按名
|
||
# 覆盖、自动化按名合并，这样两个进程各改各的任务时，后保存
|
||
# 的不会把先保存的覆盖掉。tmp 文件带 pid，避免两个进程用
|
||
# 同一个 .tmp 互相踩。锁文件在 state 文件旁边，名如
|
||
# host-state.json.lock，用 fcntl.flock（Linux/macOS 都有）。
|
||
import contextlib
|
||
try:
|
||
import fcntl
|
||
except ImportError: # Windows 等无 fcntl 的平台：退化为无锁
|
||
fcntl = None
|
||
|
||
|
||
@contextlib.contextmanager
|
||
def _state_lock(path, exclusive=True):
|
||
lock_path = path + ".lock"
|
||
os.makedirs(os.path.dirname(os.path.abspath(lock_path)), exist_ok=True)
|
||
f = open(lock_path, "a+")
|
||
try:
|
||
if fcntl is not None:
|
||
fcntl.flock(f.fileno(),
|
||
fcntl.LOCK_EX if exclusive else fcntl.LOCK_SH)
|
||
yield
|
||
finally:
|
||
if fcntl is not None:
|
||
try:
|
||
fcntl.flock(f.fileno(), fcntl.LOCK_UN)
|
||
except Exception:
|
||
pass
|
||
f.close()
|
||
|
||
|
||
def _read_state_file(path):
|
||
if os.path.exists(path):
|
||
try:
|
||
with open(path) as f:
|
||
return json.load(f)
|
||
except Exception:
|
||
pass
|
||
return {"tasks": {}, "automations": []}
|
||
|
||
|
||
def load_state_strict(path):
|
||
"""严格读 state：文件不存在返回 (空 state, None)；
|
||
文件存在但 JSON 坏了返回 (None, 错误信息)，绝不静默
|
||
当成空 state——否则损坏一次就等于全部任务丢了，
|
||
这是自动恢复要先堵住的洞（旧 _read_state_file 会吞掉
|
||
解析错误，health/recover 必须用这个严格版）。"""
|
||
if not os.path.exists(path):
|
||
return {"tasks": {}, "automations": []}, None
|
||
try:
|
||
with open(path) as f:
|
||
return json.load(f), None
|
||
except Exception as e:
|
||
return None, str(e)
|
||
|
||
|
||
def restore_state(state_path):
|
||
"""state 损坏或丢失时从备份恢复：按 .bak、.bak.1、.bak.2
|
||
顺序找第一个能解析的备份，先把坏文件（如果存在）改名
|
||
成 .corrupt.<时间戳> 留底，再把备份拷回主文件。
|
||
返回 (ok, message)。主文件完好时不做任何事。"""
|
||
strict, err = load_state_strict(state_path)
|
||
if err is None and strict is not None:
|
||
return True, "state 完好，无需恢复"
|
||
if os.path.exists(state_path):
|
||
corrupt = f"{state_path}.corrupt.{int(time.time())}"
|
||
try:
|
||
os.replace(state_path, corrupt)
|
||
except Exception as e:
|
||
return False, f"坏 state 改名留底失败：{e}"
|
||
else:
|
||
corrupt = None
|
||
for suffix in (".bak", ".bak.1", ".bak.2"):
|
||
cand = state_path + suffix
|
||
if not os.path.exists(cand):
|
||
continue
|
||
try:
|
||
with open(cand) as f:
|
||
data = json.load(f)
|
||
except Exception:
|
||
continue
|
||
import shutil
|
||
shutil.copy2(cand, state_path)
|
||
return True, (f"已从 {suffix} 恢复，version="
|
||
f"{data.get('version')}，"
|
||
f"坏文件留底：{corrupt}")
|
||
return False, "没有可用的备份（.bak/.bak.1/.bak.2 均缺失或损坏）"
|
||
|
||
|
||
def recover_interrupted(state, state_path, older_than=3600, cfg=None):
|
||
"""自动恢复（只标记，不自动重跑模型回合）：
|
||
status=running 且最后回合时间超过 older_than 秒的任务，
|
||
判定为进程被杀/VM 回收导致的中断，标 interrupted 并记日志，
|
||
用户随后可用 run 续跑（thread 在 CODEX_HOME 里不丢）。
|
||
不自动重跑是刻意的安全边界：自动重跑会不经用户同意
|
||
花中转站额度，daemon 只负责标记 + 日志 + 通知。
|
||
返回被标记的任务名列表。"""
|
||
now = time.time()
|
||
recovered = []
|
||
for name, t in (state.get("tasks") or {}).items():
|
||
if t.get("status") != "running":
|
||
continue
|
||
last = t.get("last_turn_at") or t.get("created_at") or 0
|
||
if now - last < older_than:
|
||
continue
|
||
t["status"] = "interrupted"
|
||
recovered.append(name)
|
||
try:
|
||
log_event(state_path, f"recover {name} -> interrupted "
|
||
f"(last_turn {int(now - last)}s ago)")
|
||
except Exception:
|
||
pass
|
||
if cfg is not None:
|
||
notify_failure(state_path, cfg, name,
|
||
{"status": "interrupted",
|
||
"text": "process killed / VM reclaimed"})
|
||
return recovered
|
||
|
||
|
||
def run_health_check(state_path, cfg, codex_bin, codex_home, approval,
|
||
server_factory=None):
|
||
"""健康检查（不跑模型）：返回 report dict，ok 为全部通过。
|
||
检查项：二进制存在、配置可读、state 可解析、备份存在、
|
||
daemon pid 是否存活/过期、server 能 initialize + list、
|
||
缺失 worktree（doctor 的逻辑）。server_factory 供单测注入
|
||
假 server，避免健康检查本身要拉真进程。"""
|
||
checks = {}
|
||
checks["codex_bin_exists"] = bool(codex_bin) and os.path.exists(codex_bin)
|
||
checks["config_loads"] = isinstance(cfg, dict)
|
||
strict, err = load_state_strict(state_path)
|
||
checks["state_valid"] = err is None
|
||
checks["state_error"] = err
|
||
checks["backup_exists"] = os.path.exists(state_path + ".bak")
|
||
if strict is not None:
|
||
doc = check_state(state_path, strict)
|
||
checks["failed_tasks"] = doc["failed_tasks"]
|
||
checks["missing_worktrees"] = doc["missing_worktrees"]
|
||
checks["open_review_comments"] = doc["open_review_comments"]
|
||
else:
|
||
checks["failed_tasks"] = []
|
||
checks["missing_worktrees"] = []
|
||
checks["open_review_comments"] = 0
|
||
# daemon pid
|
||
pidfile = os.path.join(os.path.dirname(os.path.abspath(state_path)),
|
||
"host-daemon.pid")
|
||
checks["daemon_pidfile"] = pidfile
|
||
if os.path.exists(pidfile):
|
||
try:
|
||
pid = int(open(pidfile).read().strip())
|
||
os.kill(pid, 0)
|
||
checks["daemon_alive"] = True
|
||
checks["daemon_pid"] = pid
|
||
except Exception:
|
||
checks["daemon_alive"] = False
|
||
checks["daemon_pid"] = None
|
||
else:
|
||
checks["daemon_alive"] = False
|
||
checks["daemon_pid"] = None
|
||
# server（无模型成本，只 initialize + list）
|
||
try:
|
||
if server_factory is not None:
|
||
srv = server_factory()
|
||
else:
|
||
srv = Server(codex_bin=codex_bin, codex_home=codex_home,
|
||
approval=approval)
|
||
try:
|
||
srv.initialize()
|
||
threads = srv.list_threads()
|
||
checks["server_ok"] = True
|
||
checks["server_threads"] = len(
|
||
(threads or {}).get("data") or []) if isinstance(threads, dict) else None
|
||
finally:
|
||
try:
|
||
srv.close()
|
||
except Exception:
|
||
pass
|
||
except Exception as e:
|
||
checks["server_ok"] = False
|
||
checks["server_error"] = str(e)[:500]
|
||
ok = bool(checks["codex_bin_exists"] and checks["state_valid"]
|
||
and checks["server_ok"])
|
||
return {"ok": ok, "checks": checks}
|
||
|
||
|
||
def daemonize(pidfile, logfile):
|
||
"""双 fork 守护进程（WeChat 桥 daemon_serve.py 同款教训：
|
||
setsid nohup 会在 launcher 会话结束时被 runtime 按进程树
|
||
回收杀掉，双 fork 到 PID 1 才能活过会话结束）。
|
||
父进程返回 False（应退出），子进程返回 True（继续跑循环）。"""
|
||
if os.fork() > 0:
|
||
return False
|
||
os.setsid()
|
||
if os.fork() > 0:
|
||
os._exit(0)
|
||
sys.stdout.flush()
|
||
sys.stderr.flush()
|
||
log_f = open(logfile, "a", encoding="utf-8")
|
||
os.dup2(log_f.fileno(), sys.stdout.fileno())
|
||
os.dup2(log_f.fileno(), sys.stderr.fileno())
|
||
with open("/dev/null", "r") as devnull:
|
||
os.dup2(devnull.fileno(), sys.stdin.fileno())
|
||
with open(pidfile, "w") as f:
|
||
f.write(str(os.getpid()))
|
||
return True
|
||
|
||
|
||
|
||
def load_state(path):
|
||
with _state_lock(path, exclusive=False):
|
||
st = _read_state_file(path)
|
||
# _loaded_version 只活在内存里，save 时不写回文件（见下），
|
||
# 用来判断“保存时磁盘版本是否比加载时新”——多机器共享
|
||
# state 时，版本跳变就是冲突信号，不静默覆盖。
|
||
st["_loaded_version"] = st.get("version", 0)
|
||
return st
|
||
|
||
|
||
def save_state(path, state):
|
||
with _state_lock(path, exclusive=True):
|
||
# 锁内重读 + 合并：内存任务覆盖同名磁盘任务，磁盘上
|
||
# 别的进程新增的任务保留；自动化按名合并，内存优先。
|
||
disk = _read_state_file(path)
|
||
loaded_version = state.get("_loaded_version", disk.get("version", 0))
|
||
if disk.get("version", 0) > loaded_version:
|
||
# 别的写者（另一进程 / 另一台机器共享文件时）已先写过，
|
||
# 合并保留了它的任务，但这是一次冲突，记日志不静默。
|
||
try:
|
||
log_path = os.path.join(
|
||
os.path.dirname(os.path.abspath(path)), "host-run.log")
|
||
ts = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
|
||
with open(log_path, "a", encoding="utf-8") as lf:
|
||
lf.write(f"{ts} STATE-CONFLICT disk_version="
|
||
f"{disk.get('version')} loaded_version="
|
||
f"{loaded_version} merged_tasks="
|
||
f"{len(disk.get('tasks') or {})}\n")
|
||
except Exception:
|
||
pass
|
||
merged_tasks = dict(disk.get("tasks") or {})
|
||
merged_tasks.update({k: v for k, v in (state.get("tasks") or {}).items()})
|
||
disk_autos = {a.get("name"): a for a in (disk.get("automations") or [])
|
||
if isinstance(a, dict)}
|
||
for a in (state.get("automations") or []):
|
||
if isinstance(a, dict) and a.get("name"):
|
||
disk_autos[a["name"]] = a
|
||
merged = {"tasks": merged_tasks,
|
||
"automations": list(disk_autos.values())}
|
||
# 保留内存 state 里 tasks/automations 之外的顶层字段，
|
||
# 但 _ 开头的内存标记不写回文件。
|
||
for k, v in state.items():
|
||
if k not in ("tasks", "automations") and not k.startswith("_"):
|
||
merged[k] = v
|
||
merged["version"] = (disk.get("version", 0) or 0) + 1
|
||
try:
|
||
import socket
|
||
merged["last_writer"] = f"{socket.gethostname()}:{os.getpid()}"
|
||
except Exception:
|
||
merged["last_writer"] = str(os.getpid())
|
||
merged["updated_at"] = time.time()
|
||
tmp = f"{path}.tmp.{os.getpid()}"
|
||
with open(tmp, "w") as f:
|
||
json.dump(merged, f, indent=2, ensure_ascii=False)
|
||
if os.path.exists(path):
|
||
import shutil
|
||
backup = path + ".bak"
|
||
backup_1 = backup + ".1"
|
||
backup_2 = backup + ".2"
|
||
backup_tmp = f"{backup}.tmp.{os.getpid()}.{threading.get_ident()}"
|
||
shutil.copy2(path, backup_tmp)
|
||
if os.path.exists(backup_2):
|
||
os.remove(backup_2)
|
||
if os.path.exists(backup_1):
|
||
os.replace(backup_1, backup_2)
|
||
if os.path.exists(backup):
|
||
os.replace(backup, backup_1)
|
||
os.replace(backup_tmp, backup)
|
||
os.replace(tmp, path)
|
||
# 让调用方内存里的 state 也跟上合并结果，避免下次保存
|
||
# 又把别的进程的任务挤掉（调用方继续用这份 state 时）。
|
||
state.clear()
|
||
state.update(merged)
|
||
state["_loaded_version"] = merged["version"]
|
||
|
||
|
||
def log_event(state_path, message):
|
||
"""run-due / merge / discard 等运维事件写日志，不依赖标准输出。
|
||
日志在 state 文件同目录的 host-run.log，定时任务跑完后
|
||
看这个文件就知道派发了什么、跳过了什么。"""
|
||
try:
|
||
log_path = os.path.join(os.path.dirname(os.path.abspath(state_path)),
|
||
"host-run.log")
|
||
ts = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
|
||
with open(log_path, "a", encoding="utf-8") as f:
|
||
f.write(f"{ts} {message}\n")
|
||
except Exception:
|
||
pass
|
||
|
||
|
||
def git(args, cwd):
|
||
return subprocess.run(["git"] + args, cwd=cwd, capture_output=True,
|
||
text=True)
|
||
|
||
|
||
def make_worktree(repo, task_name):
|
||
"""一任务一 worktree；不是 git 仓库时退化为独立目录拷贝。"""
|
||
repo = os.path.abspath(repo)
|
||
base = os.path.join(HERE, "work", "host-worktrees")
|
||
os.makedirs(base, exist_ok=True)
|
||
path = os.path.join(base, task_name)
|
||
if os.path.exists(path):
|
||
return path, None
|
||
if os.path.isdir(os.path.join(repo, ".git")):
|
||
branch = f"host/{task_name}"
|
||
r = git(["worktree", "add", path, "-b", branch, "HEAD"], repo)
|
||
if r.returncode != 0:
|
||
# 分支已存在时直接用它
|
||
r = git(["worktree", "add", path, branch], repo)
|
||
if r.returncode != 0:
|
||
raise RuntimeError(f"git worktree 失败: {r.stderr}")
|
||
return path, branch
|
||
# 非 git：复制一份
|
||
import shutil
|
||
shutil.copytree(repo, path)
|
||
return path, None
|
||
|
||
|
||
def task_diff(worktree):
|
||
if os.path.isdir(os.path.join(worktree, ".git")) or \
|
||
subprocess.run(["git", "-C", worktree, "rev-parse"],
|
||
capture_output=True).returncode == 0:
|
||
st = git(["status", "--porcelain"], worktree).stdout
|
||
diff = git(["diff", "HEAD"], worktree).stdout
|
||
untracked = git(["ls-files", "--others", "--exclude-standard"],
|
||
worktree).stdout
|
||
# 生产评审要点：新文件默认是 untracked，git diff 里看不到内容，
|
||
# 只列文件名等于没评审。把未追踪文本文件的内容也带上（截断）。
|
||
contents = {}
|
||
for rel in untracked.split():
|
||
p = os.path.join(worktree, rel)
|
||
if rel.endswith(".pyc") or "__pycache__" in rel:
|
||
continue
|
||
try:
|
||
with open(p, encoding="utf-8") as f:
|
||
contents[rel] = f.read(8000)
|
||
except Exception:
|
||
contents[rel] = "<binary or unreadable>"
|
||
return {"status": st, "diff": diff, "untracked": untracked,
|
||
"untracked_contents": contents}
|
||
return {"status": "", "diff": "",
|
||
"untracked": "\n".join(sorted(os.listdir(worktree)))}
|
||
|
||
|
||
# ---------------- 第二类缺口补齐：依赖 / 验测 / 评审评论 / 预算 ----------------
|
||
|
||
def topo_sort_jobs(jobs):
|
||
"""依赖拓扑 + 优先级：依赖先跑，同层按 priority 降序、名字升序。
|
||
缺依赖或成环时抛 ValueError，不静默跳过。"""
|
||
by_name = {}
|
||
for j in jobs:
|
||
by_name[j["name"]] = j
|
||
for j in jobs:
|
||
for d in (j.get("depends_on") or []):
|
||
if d not in by_name:
|
||
raise ValueError(f"任务 {j['name']} 依赖不存在的任务 {d}")
|
||
if d == j["name"]:
|
||
raise ValueError(f"任务 {j['name']} 不能依赖自己")
|
||
ordered = []
|
||
done = set()
|
||
remaining = list(jobs)
|
||
while remaining:
|
||
ready = [j for j in remaining
|
||
if all(d in done for d in (j.get("depends_on") or []))]
|
||
if not ready:
|
||
names = ", ".join(j["name"] for j in remaining)
|
||
raise ValueError(f"依赖成环，无法排序：{names}")
|
||
ready.sort(key=lambda j: (-(j.get("priority") or 0), j["name"]))
|
||
# 一波只取优先级最高的那批？不，一波取全部 ready，
|
||
# 这样同层可以并行，优先级只决定同层内的启动顺序。
|
||
# 但为了“高优先级先完成”可观测，返回顺序即启动顺序。
|
||
for j in ready:
|
||
ordered.append(j)
|
||
done.add(j["name"])
|
||
remaining = [j for j in remaining if j["name"] not in done]
|
||
return ordered
|
||
|
||
|
||
def detect_verify_command(repo):
|
||
"""合并后验测的默认命令：仓库有 tests/ 就跑标准库 unittest，
|
||
没有就返回 None（不猜别的测试框架，不强制跑）。"""
|
||
if repo and os.path.isdir(os.path.join(repo, "tests")):
|
||
return "python3 -m unittest discover -s tests -v"
|
||
return None
|
||
|
||
|
||
def run_verify(command, cwd):
|
||
if not command:
|
||
return 0, ""
|
||
v = subprocess.run(command, shell=True, cwd=cwd,
|
||
capture_output=True, text=True, timeout=600)
|
||
return v.returncode, (v.stdout + v.stderr).strip()[:2000]
|
||
|
||
|
||
def merge_task(repo, worktree, branch, task_name, verify_command=None):
|
||
"""合并 + 验测 + 回滚（纯 git 操作，不依赖 server，可单测）。
|
||
返回 (ok, message)：验测失败时 reset 回合并前 HEAD，分支和
|
||
worktree 保留给修复；验测通过时才删 worktree/分支。"""
|
||
if verify_command is None:
|
||
verify_command = detect_verify_command(repo)
|
||
pre_head = git(["rev-parse", "HEAD"], repo).stdout.strip()
|
||
# 合并前先清掉 worktree 里的 __pycache__ / *.pyc：agent 跑
|
||
# 测试时会生成它们，git add -A 会把二进制缓存也提交进主仓，
|
||
# 下次合并时主仓的 .pyc 被改动就会挡住 merge（gap3 实测）。
|
||
# .pyc 是缓存，删了可再生，安全。
|
||
subprocess.run(["find", worktree, "-type", "d", "-name", "__pycache__",
|
||
"-exec", "rm", "-rf", "{}", "+"],
|
||
capture_output=True)
|
||
subprocess.run(["find", worktree, "-name", "*.pyc", "-delete"],
|
||
capture_output=True)
|
||
# 主仓里已入库的 .pyc 若被测试改动，也先还原，否则 merge
|
||
# 会报 local changes would be overwritten。
|
||
git(["checkout", "--", "*.pyc"], repo)
|
||
git(["add", "-A"], worktree)
|
||
c = git(["commit", "-m", f"host: {task_name}"], worktree)
|
||
if c.returncode != 0 and "nothing to commit" not in c.stdout + c.stderr:
|
||
return False, f"worktree 提交失败：{c.stderr}"
|
||
r = git(["merge", "--no-ff", branch, "-m", f"host: {task_name}"], repo)
|
||
if r.returncode != 0:
|
||
git(["merge", "--abort"], repo)
|
||
# 依赖任务的常见冲突：依赖任务的分支里已经包含依赖的改动，
|
||
# 依赖先合并后，再合并依赖任务会在同一位置重复加同一段
|
||
# 代码（gap2 实测：add2 在两边都出现）。这种“重复加同一段”
|
||
# 可以用 -X ours 自动解：主仓版本优先，任务的新增部分
|
||
# （mul2、add2_mul2 等）仍会合进来；解完后必须逐行核对
|
||
# 任务分支的新增行是否都在主仓里，缺一行就回滚，不静默丢改动。
|
||
r2 = git(["merge", "--no-ff", "-X", "ours", branch,
|
||
"-m", f"host: {task_name}"], repo)
|
||
if r2.returncode != 0:
|
||
git(["merge", "--abort"], repo)
|
||
return False, f"合并冲突，已 abort：{r.stderr}"
|
||
# 核对新增行：以 merge-base 到分支的 diff 为准
|
||
base = git(["merge-base", pre_head, branch], repo).stdout.strip()
|
||
diff = git(["diff", base, branch], repo).stdout
|
||
added = [ln[1:].strip() for ln in diff.splitlines()
|
||
if ln.startswith("+") and not ln.startswith("+++")
|
||
and ln[1:].strip()]
|
||
# 合并后的工作区文件内容（主仓最新）
|
||
missing = []
|
||
for ln in added:
|
||
found = False
|
||
for root, dirs, files in os.walk(repo):
|
||
dirs[:] = [d for d in dirs if d not in (".git", "__pycache__")]
|
||
for fn in files:
|
||
if fn.endswith(".pyc"):
|
||
continue
|
||
try:
|
||
with open(os.path.join(root, fn), encoding="utf-8",
|
||
errors="ignore") as f:
|
||
if ln in f.read():
|
||
found = True
|
||
break
|
||
except Exception:
|
||
continue
|
||
if found:
|
||
break
|
||
if not found:
|
||
missing.append(ln)
|
||
if missing:
|
||
git(["reset", "--hard", pre_head], repo)
|
||
return False, (f"合并冲突自动解后发现任务改动被丢弃 "
|
||
f"（缺 {len(missing)} 行，如 {missing[:3]}），"
|
||
f"已回滚到 {pre_head[:8]}，需手动解冲突。")
|
||
# 自动解成功，继续走下面的验测（不 return，落到验测分支）
|
||
print(f"注意：合并冲突用 -X ours 自动解（重复加同一段），"
|
||
f"新增行 {len(added)} 行已核对都在主仓里。")
|
||
if verify_command:
|
||
rc, out = run_verify(verify_command, repo)
|
||
if rc != 0:
|
||
git(["reset", "--hard", pre_head], repo)
|
||
return False, (f"合并后验测失败，已回滚到 {pre_head[:8]}，"
|
||
f"分支 {branch} 和 worktree 保留：\n{out[:1000]}")
|
||
git(["worktree", "remove", "--force", worktree], repo)
|
||
git(["branch", "-d", branch], repo)
|
||
return True, f"merged {branch} into {repo}"
|
||
|
||
|
||
def add_review_comment(task, file_path, line, comment):
|
||
task.setdefault("review_comments", [])
|
||
task["review_comments"].append({
|
||
"file": file_path, "line": int(line), "comment": comment,
|
||
"status": "open", "created_at": time.time()})
|
||
return task["review_comments"][-1]
|
||
|
||
|
||
def open_review_comments(task):
|
||
return [c for c in (task.get("review_comments") or [])
|
||
if c.get("status") == "open"]
|
||
|
||
|
||
def build_fix_prompt(task):
|
||
comments = open_review_comments(task)
|
||
lines = ["请按以下评审评论逐条修复，只改评论涉及的文件和行附近，"
|
||
"不要改无关文件，修完后简述每条评论的处理结果："]
|
||
for c in comments:
|
||
lines.append(f"- {c['file']}:{c['line']}: {c['comment']}")
|
||
return "\n".join(lines)
|
||
|
||
|
||
def record_turn_usage(state, failed=False):
|
||
today = time.strftime("%Y-%m-%d", time.localtime())
|
||
usage = state.get("usage") or {}
|
||
if usage.get("date") != today:
|
||
usage = {"date": today, "turns": 0, "failed_turns": 0}
|
||
usage["turns"] = usage.get("turns", 0) + 1
|
||
if failed:
|
||
usage["failed_turns"] = usage.get("failed_turns", 0) + 1
|
||
state["usage"] = usage
|
||
return usage
|
||
|
||
|
||
def budget_exceeded(state, cfg):
|
||
"""成本预算：app-server 协议不暴露真实 token 用量，
|
||
所以预算按 turn 数算（不是 token 数），这是诚实边界。
|
||
cfg 里 max_daily_turns 未配时不限流。"""
|
||
limit = None
|
||
try:
|
||
limit = int(cfg.get("max_daily_turns")) if cfg.get("max_daily_turns") else None
|
||
except Exception:
|
||
limit = None
|
||
if not limit:
|
||
return False
|
||
usage = state.get("usage") or {}
|
||
today = time.strftime("%Y-%m-%d", time.localtime())
|
||
if usage.get("date") != today:
|
||
return False
|
||
return usage.get("turns", 0) >= limit
|
||
|
||
|
||
def notify_failure(state_path, cfg, task_name, result):
|
||
"""失败通知：先写 host-run.log（一定有），再跑用户配置的
|
||
notify_command（本地 shell 命令，只来自用户自己的配置，
|
||
不是外部输入；模板里可用 {task} {status} {text}）。
|
||
|
||
插进命令串的值一律 shlex.quote：模型文本可能含 shell 元
|
||
字符（引号、分号、$() 等），不加 quote 就等于把模型输出
|
||
当命令执行。模板本身有问题（未知占位符、未成对的大括号）
|
||
时不再静默吞掉：记一条 template error 到 host-run.log 并
|
||
且不跑这条命令。命令的返回值 / 超时 / 异常同样记日志，
|
||
否则通知丢了都无从发现（旧行为是异常全吞、无任何记录）。
|
||
没有配 notify_command 时只写 host-run.log，不外发。"""
|
||
result = result or {}
|
||
text = result.get("text", "") or ""
|
||
status = result.get("status", "") or ""
|
||
msg = f"FAILED {task_name}: {status} {text[:200]}"
|
||
try:
|
||
log_event(state_path, msg)
|
||
except Exception:
|
||
pass
|
||
cmd = (cfg or {}).get("notify_command")
|
||
if not cmd:
|
||
return msg
|
||
try:
|
||
formatted = cmd.format(task=shlex.quote(str(task_name)),
|
||
status=shlex.quote(str(status)),
|
||
text=shlex.quote(str(text[:200])))
|
||
except (KeyError, IndexError, ValueError) as exc:
|
||
# 模板写错（例如 {unknown} 或单个 {）时 .format 抛错：
|
||
# 明确记日志、不跑命令、不让调用方以为通知已发出。
|
||
try:
|
||
log_event(state_path,
|
||
"notify command template error: "
|
||
f"{type(exc).__name__}: {exc}")
|
||
except Exception:
|
||
pass
|
||
return msg
|
||
try:
|
||
proc = subprocess.run(formatted, shell=True, timeout=30)
|
||
try:
|
||
log_event(state_path,
|
||
f"notify command rc={proc.returncode} "
|
||
f"task={task_name}")
|
||
except Exception:
|
||
pass
|
||
except subprocess.TimeoutExpired:
|
||
try:
|
||
log_event(state_path,
|
||
f"notify command timeout task={task_name}")
|
||
except Exception:
|
||
pass
|
||
except Exception as exc:
|
||
try:
|
||
log_event(state_path,
|
||
"notify command failed: "
|
||
f"{type(exc).__name__} task={task_name}")
|
||
except Exception:
|
||
pass
|
||
return msg
|
||
|
||
|
||
def check_state(state_path, state):
|
||
"""doctor：state 版本、备份、失败任务、缺失 worktree。
|
||
不跑模型，不改 state。"""
|
||
out = {"state_path": state_path,
|
||
"version": state.get("version"),
|
||
"last_writer": state.get("last_writer"),
|
||
"updated_at": state.get("updated_at"),
|
||
"usage": state.get("usage"),
|
||
"backup_exists": os.path.exists(state_path + ".bak"),
|
||
"failed_tasks": [n for n, t in (state.get("tasks") or {}).items()
|
||
if t.get("status") in ("failed", "goal-unmet",
|
||
"merge-failed")],
|
||
"missing_worktrees": [n for n, t in (state.get("tasks") or {}).items()
|
||
if t.get("cwd") and not os.path.isdir(t["cwd"])
|
||
and t.get("status") not in ("merged", "discarded", "archived")],
|
||
"open_review_comments": sum(
|
||
len(open_review_comments(t))
|
||
for t in (state.get("tasks") or {}).values())}
|
||
return out
|
||
|
||
|
||
# ---------------- GitHub PR / CI ----------------
|
||
# 这一层只做 host -> GitHub 的桥，不做本地合并：本地合并仍是
|
||
# merge_task()，PR 合并由 GitHub 完成，两条线互不干扰。
|
||
# 真实 PR 需要两样外部条件，缺一不可，且都不由本轮代码提供：
|
||
# 1. 仓库有 git remote origin（本仓目前没有 remote）；
|
||
# 2. gh 已登录（gh auth login，PAT 走 Secure Vault，
|
||
# 绝不写进文件、state、日志或聊天记录）。
|
||
# gh 调用全部走 run_gh()，单测注入假 gh，不依赖网络和登录。
|
||
|
||
HOST_VERIFY_WORKFLOW = """name: host-verify
|
||
on:
|
||
pull_request:
|
||
push:
|
||
branches: [main]
|
||
jobs:
|
||
verify:
|
||
runs-on: ubuntu-latest
|
||
steps:
|
||
- uses: actions/checkout@v4
|
||
- uses: actions/setup-python@v5
|
||
with:
|
||
python-version: "3.12"
|
||
- run: python3 -m unittest discover -s tests -v
|
||
- run: python3 -m unittest discover -s cloud/tests -v
|
||
"""
|
||
|
||
|
||
def generate_github_workflow(repo):
|
||
"""生成 .github/workflows/host-verify.yml（幂等）：
|
||
返回 (changed, path)。文件已存在且内容相同则不改写。"""
|
||
path = os.path.join(repo, ".github", "workflows", "host-verify.yml")
|
||
os.makedirs(os.path.dirname(path), exist_ok=True)
|
||
if os.path.exists(path):
|
||
with open(path, encoding="utf-8") as f:
|
||
if f.read() == HOST_VERIFY_WORKFLOW:
|
||
return False, path
|
||
with open(path, "w", encoding="utf-8") as f:
|
||
f.write(HOST_VERIFY_WORKFLOW)
|
||
return True, path
|
||
|
||
|
||
def run_gh(args, cwd=None, runner=None):
|
||
"""gh 调用唯一入口。runner 供单测注入：runner(args) 返回
|
||
一个带 returncode/stdout/stderr 的对象；真实路径走 gh CLI，
|
||
cwd 传给 gh 找仓库上下文（假 runner 忽略 cwd）。"""
|
||
if runner is not None:
|
||
return runner(args)
|
||
return subprocess.run(["gh"] + list(args), cwd=cwd,
|
||
capture_output=True, text=True)
|
||
|
||
|
||
def _github_remote_url(repo):
|
||
r = git(["remote", "get-url", "origin"], repo)
|
||
url = (r.stdout or "").strip()
|
||
return url if r.returncode == 0 and url else None
|
||
|
||
|
||
def build_pr_body(task):
|
||
"""PR 正文：任务名、提示摘要、未解决评审评论。
|
||
只含任务本身的信息，不含密钥。"""
|
||
prompt = (task.get("prompt") or "")[:500]
|
||
lines = [f"host 任务：{task.get('name')}",
|
||
"",
|
||
"任务提示摘要：",
|
||
prompt or "（无提示）",
|
||
""]
|
||
comments = open_review_comments(task)
|
||
if comments:
|
||
lines.append("评审评论（未解决）：")
|
||
for c in comments:
|
||
lines.append(f"- {c['file']}:{c['line']}: {c['comment']}")
|
||
else:
|
||
lines.append("评审评论：无未解决评论")
|
||
return "\n".join(lines)
|
||
|
||
|
||
def create_pr(repo, task, gh_runner=None):
|
||
"""建 PR：先校验任务，再 push 分支，再 gh pr create。
|
||
成功返回 {pr_url, pr_number, branch} 并写回任务记录；
|
||
任何一步失败都抛异常，不编造 URL、不静默失败。
|
||
调用方负责 save_state。"""
|
||
branch = task.get("branch")
|
||
if not branch:
|
||
raise ValueError(f"任务 {task.get('name')} 没有 branch，"
|
||
f"无法建 PR（可能是没用 worktree 派发的任务）")
|
||
if task.get("status") not in ("completed", "merged"):
|
||
raise ValueError(f"任务 {task.get('name')} 状态是 "
|
||
f"{task.get('status')}，只有 completed/merged "
|
||
f"才能建 PR")
|
||
if not _github_remote_url(repo):
|
||
raise RuntimeError(
|
||
"仓库没有 git remote origin，无法 push 分支和建 PR："
|
||
"请先给仓库加 remote（git remote add origin <url>）"
|
||
"并用 gh auth login 登录，PAT 走 Secure Vault，"
|
||
"不要写进文件或聊天记录")
|
||
push = git(["push", "origin", branch], repo)
|
||
if push.returncode != 0:
|
||
raise RuntimeError(f"git push origin {branch} 失败："
|
||
f"{push.stderr or push.stdout}")
|
||
title = f"host: {task.get('name')}"
|
||
r = run_gh(["pr", "create", "--title", title,
|
||
"--body", build_pr_body(task), "--head", branch],
|
||
cwd=repo, runner=gh_runner)
|
||
out = (r.stdout or "").strip()
|
||
if r.returncode != 0 or not out:
|
||
# 失败时打印 gh 的错误原文，不编造 URL。
|
||
raise RuntimeError(f"gh pr create 失败：{r.stderr or out}")
|
||
pr_url = out.splitlines()[0].strip()
|
||
m = re.search(r"/(\d+)\s*$", pr_url)
|
||
pr_number = int(m.group(1)) if m else None
|
||
task["pr_url"] = pr_url
|
||
task["pr_number"] = pr_number
|
||
return {"pr_url": pr_url, "pr_number": pr_number, "branch": branch}
|
||
|
||
|
||
def parse_pr_checks(output):
|
||
"""解析 gh pr checks 输出，兼容两种常见格式：
|
||
制表符格式（name<TAB>state…）和人类可读格式（✓ build pass）。
|
||
返回 (checks, all_pass)：checks 为 [{name, state}]，state
|
||
统一成 pass/fail/pending/skipped/cancelled/neutral；
|
||
没有任何 check 时 all_pass 为 False（空 PR 不能算全过）。"""
|
||
checks = []
|
||
for raw in (output or "").splitlines():
|
||
line = raw.strip()
|
||
if not line or "no checks" in line.lower():
|
||
continue
|
||
if line.lower().startswith("name"):
|
||
continue
|
||
if "\t" in line:
|
||
parts = [p.strip() for p in line.split("\t") if p.strip()]
|
||
if len(parts) < 2:
|
||
continue
|
||
name, state_raw = parts[0], parts[1].lower()
|
||
else:
|
||
m = re.search(r"\b(pass|passing|fail|failing|failed|pending|"
|
||
r"queued|waiting|in_progress|skipped|cancelled|"
|
||
r"neutral)\b", line.lower())
|
||
if not m:
|
||
continue
|
||
state_raw = m.group(1)
|
||
name = line[:m.start()].strip(" ✓✗●○-").strip()
|
||
if not name:
|
||
name = line
|
||
state = {"passing": "pass", "failing": "fail", "failed": "fail",
|
||
"queued": "pending", "waiting": "pending",
|
||
"in_progress": "pending"}.get(state_raw, state_raw)
|
||
checks.append({"name": name, "state": state})
|
||
all_pass = bool(checks) and all(c["state"] == "pass" for c in checks)
|
||
return checks, all_pass
|
||
|
||
|
||
def get_pr_checks(task, gh_runner=None, repo=None):
|
||
"""查 PR 的 CI 状态。任务须已有 pr_number，否则报错。"""
|
||
number = task.get("pr_number")
|
||
if not number:
|
||
raise ValueError(f"任务 {task.get('name')} 还没有 PR 编号，"
|
||
f"先跑 pr 建 PR")
|
||
r = run_gh(["pr", "checks", str(number)], cwd=repo, runner=gh_runner)
|
||
if r.returncode != 0:
|
||
raise RuntimeError(f"gh pr checks 失败：{r.stderr or r.stdout}")
|
||
checks, all_pass = parse_pr_checks(r.stdout or "")
|
||
return {"pr_url": task.get("pr_url"), "checks": checks,
|
||
"all_pass": all_pass}
|
||
|
||
|
||
def merge_pr(task, gh_runner=None, repo=None):
|
||
"""PR 合并门：先查 checks，未全过拒绝且不调 merge；
|
||
全过后调 gh pr merge --merge，成功才把任务标 merged。
|
||
不动本地主分支——合并由 GitHub 完成，本地分支/主仓
|
||
不由这条命令改（和本地 merge_task() 是两条线）。"""
|
||
info = get_pr_checks(task, gh_runner=gh_runner, repo=repo)
|
||
if not info["all_pass"]:
|
||
bad = [c for c in info["checks"] if c["state"] != "pass"]
|
||
return False, (f"PR checks 未全过，拒绝合并："
|
||
f"{[(c['name'], c['state']) for c in bad]}")
|
||
r = run_gh(["pr", "merge", str(task["pr_number"]), "--merge"],
|
||
cwd=repo, runner=gh_runner)
|
||
if r.returncode != 0:
|
||
return False, f"gh pr merge 失败：{r.stderr or r.stdout}"
|
||
task["status"] = "merged"
|
||
return True, f"PR #{task['pr_number']} 已合并：{task.get('pr_url')}"
|
||
|
||
|
||
|
||
|
||
def copy_tree_excluding_git(src, dst):
|
||
"""把 src 工作区的文件拷到 dst（不含 .git、__pycache__、.pyc），
|
||
fork 和依赖任务都用它：依赖任务要站在依赖任务的改动上，
|
||
而不是只站在主仓 HEAD 上，否则依赖只是“等一等”，代码上
|
||
根本看不到依赖的改动。"""
|
||
import shutil
|
||
for root, dirs, files in os.walk(src):
|
||
dirs[:] = [d for d in dirs if d not in (".git", "__pycache__")]
|
||
for fn in files:
|
||
if fn.endswith(".pyc") or fn == ".git":
|
||
continue
|
||
s = os.path.join(root, fn)
|
||
rel = os.path.relpath(s, src)
|
||
d = os.path.join(dst, rel)
|
||
os.makedirs(os.path.dirname(d), exist_ok=True)
|
||
shutil.copy2(s, d)
|
||
|
||
|
||
def dispatch_from_branch(server, state, name, prompt, repo, dep_branch,
|
||
dep_worktree, model, effort, sandbox,
|
||
approval_policy, depends_on=None, priority=0,
|
||
max_retries=0):
|
||
"""依赖任务的正确派发：先把依赖任务 worktree 的改动提交到
|
||
它的分支，再从该分支建新 worktree。这样依赖任务的分支是
|
||
依赖分支的后代，合并时先合依赖、再合依赖任务就不会出现
|
||
“两边在同一位置重复加同一段代码”的冲突（gap2 实测：
|
||
文件拷贝法建的分支没有这个血缘关系，合并时必然冲突，
|
||
-X ours 还会把依赖任务独有的改动整段丢掉，幸好验测
|
||
核对把它拦下来回滚了）。"""
|
||
repo = os.path.abspath(repo)
|
||
git(["add", "-A"], dep_worktree)
|
||
c = git(["commit", "-m", f"host: {name} base from {dep_branch}"],
|
||
dep_worktree)
|
||
# nothing to commit 时分支里已有全部改动，继续用它当基底
|
||
base = os.path.join(HERE, "work", "host-worktrees")
|
||
os.makedirs(base, exist_ok=True)
|
||
path = os.path.join(base, name)
|
||
if os.path.exists(path):
|
||
return None, "worktree 已存在"
|
||
branch = f"host/{name}"
|
||
r = git(["worktree", "add", path, "-b", branch, dep_branch], repo)
|
||
if r.returncode != 0:
|
||
return None, f"git worktree from {dep_branch} 失败: {r.stderr}"
|
||
thread_id = server.create_thread(path, model=model, sandbox=sandbox,
|
||
approval_policy=approval_policy)
|
||
task = {"name": name, "thread_id": thread_id, "cwd": path,
|
||
"repo": repo, "branch": branch, "sandbox": sandbox,
|
||
"status": "running", "prompt": prompt, "parent_task": None,
|
||
"depends_on": list(depends_on or []), "priority": int(priority or 0),
|
||
"max_retries": int(max_retries or 0), "retry_count": 0,
|
||
"created_at": time.time(), "turns": []}
|
||
state["tasks"][name] = task
|
||
return task, ""
|
||
|
||
|
||
def _seed_worktree(cwd, seed_dir):
|
||
"""把暂存附件拷进 worktree（dispatch/goal 共用，turn 开始前）。
|
||
|
||
seed_dir 是服务端写好的暂存目录（只含常规文件，无符号链接）：
|
||
顶层 .git 与符号链接一律跳过；子目录整树拷入，同名文件覆盖
|
||
（worktree 刚从 repo 建出来，覆盖即用户文件优先）。"""
|
||
import shutil
|
||
seed_dir = os.path.abspath(seed_dir)
|
||
if not os.path.isdir(seed_dir):
|
||
raise RuntimeError(f"seed-dir 不是目录: {seed_dir}")
|
||
for entry in sorted(os.listdir(seed_dir)):
|
||
if entry == ".git":
|
||
continue
|
||
src = os.path.join(seed_dir, entry)
|
||
dst = os.path.join(cwd, entry)
|
||
if os.path.islink(src):
|
||
continue
|
||
if os.path.isdir(src):
|
||
shutil.copytree(src, dst, dirs_exist_ok=True, symlinks=False)
|
||
else:
|
||
shutil.copy2(src, dst)
|
||
|
||
|
||
def dispatch(server, state, name, prompt, repo, model, effort, sandbox,
|
||
approval_policy, use_worktree=True, parent_task=None,
|
||
depends_on=None, priority=0, max_retries=0, seed_dir=None):
|
||
if name in state["tasks"]:
|
||
raise RuntimeError(f"task 已存在: {name}，先换名或 discard 它")
|
||
if repo and not os.path.isabs(repo) and not os.path.exists(repo):
|
||
cand = os.path.join(HERE, repo)
|
||
if os.path.exists(cand):
|
||
repo = cand
|
||
if not sandbox:
|
||
sandbox = "workspace-write"
|
||
cwd = repo
|
||
branch = None
|
||
if use_worktree and repo:
|
||
cwd, branch = make_worktree(repo, name)
|
||
if seed_dir:
|
||
# 附件只能进 worktree：直接以 repo 为 cwd 时拷进去等于污染
|
||
# 仓库，必须 loud fail，绝不静默放行。
|
||
if not use_worktree or not repo or not cwd:
|
||
raise RuntimeError(
|
||
"seed-dir 需要 worktree，当前派发没有建 worktree")
|
||
_seed_worktree(cwd, seed_dir)
|
||
thread_id = server.create_thread(cwd, model=model, sandbox=sandbox,
|
||
approval_policy=approval_policy)
|
||
task = {"name": name, "thread_id": thread_id, "cwd": cwd,
|
||
"repo": os.path.abspath(repo) if repo else None,
|
||
"branch": branch, "sandbox": sandbox, "status": "running",
|
||
"prompt": prompt, "parent_task": parent_task,
|
||
"depends_on": list(depends_on or []), "priority": int(priority or 0),
|
||
"max_retries": int(max_retries or 0), "retry_count": 0,
|
||
"created_at": time.time(), "turns": []}
|
||
state["tasks"][name] = task
|
||
return task
|
||
|
||
|
||
def run_task_turn(server, state, task, prompt, model, effort,
|
||
sandbox=None, timeout=None, interrupt_file=None):
|
||
# task 自带 sandbox 时优先用它（只读评审任务不能被全局默认覆盖），
|
||
# 否则用调用方显式传的 sandbox；两者都没有时不传 sandboxPolicy，
|
||
# 保留 thread 原有设置（旧任务没有 sandbox 记录时尤其重要）。
|
||
# timeout 为 None 时不传给 server（沿用 server 默认 600），
|
||
# 这样旧 FakeServer（不接受 timeout 参数）不受影响；显式
|
||
# 传了才透传，0 由 server 侧解释为不限时。interrupt_file 同理：
|
||
# 不传时不进 server 参数，旧 FakeServer 不受影响；传了时
|
||
# 属主在等 turn 时每秒轮询它（见 _interrupt_request_path）。
|
||
use_sandbox = task.get("sandbox") or sandbox
|
||
started = time.time()
|
||
turn_kwargs = {}
|
||
if timeout is not None:
|
||
turn_kwargs["timeout"] = timeout
|
||
if interrupt_file is not None:
|
||
turn_kwargs["interrupt_file"] = interrupt_file
|
||
try:
|
||
result = server.run_turn(task["thread_id"], prompt, model=model,
|
||
effort=effort, sandbox=use_sandbox,
|
||
**turn_kwargs)
|
||
except TypeError as exc:
|
||
# 兼容旧 server：不接受 interrupt_file 参数时退回不传它
|
||
# 再跑一次（goal 单测的 FakeServer 就是这款）；其它
|
||
# TypeError 一律不吞，原样抛出。
|
||
if "interrupt_file" not in str(exc):
|
||
raise
|
||
turn_kwargs.pop("interrupt_file", None)
|
||
result = server.run_turn(task["thread_id"], prompt, model=model,
|
||
effort=effort, sandbox=use_sandbox,
|
||
**turn_kwargs)
|
||
finished = time.time()
|
||
record_turn_usage(state, failed=(result["status"] != "completed"))
|
||
task["turns"].append({"prompt": prompt, "result": result,
|
||
"started_at": started, "finished_at": finished})
|
||
task["last_turn_at"] = finished
|
||
# 对齐 Codex App：超时 / 被中断不是失败，是可续跑的中断态；
|
||
# 只有真正的 failed 才记 failed，recover 才能区分两者。
|
||
if result["status"] == "completed":
|
||
task["status"] = "completed"
|
||
elif result["status"] in ("timeout", "interrupted"):
|
||
task["status"] = "interrupted"
|
||
else:
|
||
task["status"] = "failed"
|
||
task["last_text"] = result.get("text", "")
|
||
return result
|
||
|
||
|
||
def run_task_turn_with_retry(server, state, task, prompt, model, effort,
|
||
sandbox=None, cfg=None, timeout=None,
|
||
interrupt_file=None):
|
||
"""失败自动重试：默认 max_retries=0 时行为不变（只跑一次）；
|
||
配了重试时，失败后同 prompt 再跑，最多 max_retries 次，
|
||
每次重试前先查预算，超预算直接停，不烧钱。"""
|
||
max_retries = int(task.get("max_retries") or 0)
|
||
result = None
|
||
usage = None
|
||
for attempt in range(max_retries + 1):
|
||
if attempt > 0:
|
||
task["retry_count"] = attempt
|
||
prompt_try = (f"{prompt}\n\n（第 {attempt} 次重试："
|
||
f"上一轮状态 {result.get('status')}，"
|
||
f"请针对失败原因重试，不要重复已完成的部分。）")
|
||
else:
|
||
prompt_try = prompt
|
||
if cfg is not None and budget_exceeded(state, cfg):
|
||
result = {"status": "failed", "text": "",
|
||
"error": "budget exceeded: max_daily_turns"}
|
||
task["status"] = "failed"
|
||
task["last_text"] = ""
|
||
break
|
||
result = run_task_turn(server, state, task, prompt_try,
|
||
model, effort, sandbox, timeout=timeout,
|
||
interrupt_file=interrupt_file)
|
||
# 各次尝试的真实用量都要算（失败尝试花的钱也是钱），
|
||
# 最后一次的结果里带汇总，不只带最后一次。
|
||
usage = _add_usage(usage, result.get("usage"))
|
||
if result["status"] == "completed":
|
||
break
|
||
if usage is not None and result is not None:
|
||
result = dict(result, usage=usage)
|
||
return result
|
||
|
||
|
||
GOAL_DONE_MARKER = "目标已达成"
|
||
|
||
|
||
def _llm_declared_done(text):
|
||
"""模型是否宣告目标达成：回合文本里单独一行是标记行。
|
||
|
||
只有完整跑完的回合才查标记（调用方保证）：超时/中断回合的
|
||
部分文本即使含标记也不算，否则没做完也会提前结束。
|
||
|
||
行尾句号要容忍：提示本身让模型输出“目标已达成。”（带句
|
||
号），若检测要求整行恰好等于无句号的标记，模型照做也永
|
||
远不算达成、循环空转（2026-10-27 实弹 292 轮教训）。所以
|
||
比对前先去行尾空白与句末标点，但行内其它文字一律不算
|
||
（“xx目标已达成”“目标已达成，继续”仍不算，单独一行要
|
||
求不动）。"""
|
||
if not text:
|
||
return False
|
||
punct = "。！!？?；;，,、.．"
|
||
for line in text.splitlines():
|
||
if line.strip().rstrip(punct).strip() == GOAL_DONE_MARKER:
|
||
return True
|
||
return False
|
||
|
||
|
||
def run_goal_loop(server, state, state_path, task, goal_text, verify,
|
||
max_rounds, model, effort, sandbox, cfg=None,
|
||
timeout=None, verify_timeout=600, start_round=1,
|
||
last_verify="", last_turn_status=None, last_turn_text=""):
|
||
"""目标循环（长任务版，对齐 Codex App 的可续跑语义）。
|
||
|
||
两种验收形态（2026-10-27 起，console goal 模式走新形态）：
|
||
旧形态 verify 非空：每轮 turn → 跑验收命令 → v_rc==0 达标，
|
||
max_rounds 穷尽则 unmet。
|
||
新形态 verify 为空：不跑验收命令，达标与否由模型自己在回合
|
||
末尾单独一行输出“目标已达成”宣告；max_rounds=0 表示不限轮次，
|
||
一直跑到模型宣告、预算耗尽或用户取消为止（与 turn_timeout=0
|
||
不限时同约定）。轮次提示里会把标记约定写明，模型才知道怎么
|
||
宣告；同一 thread 保留历史，续轮提示可以很短。
|
||
每轮结束立即落盘（检查点），进程被杀 / VM 回收后可用
|
||
goal-resume 从 goal_rounds_done 的下一轮续跑，而不是从头重来。
|
||
与云端 worker 的解析契约（cloud/sandbox.py）：
|
||
结论行必须是 "goal met after rounds" / "goal unmet after rounds"，
|
||
每轮行必须是 "round N: turn=... verify_rc=... verify_out=..."，
|
||
（无验收时 verify_rc 记 -，verify_out 为空），
|
||
用量行必须是 "usage input=... cached=... ..." 格式，改动时
|
||
不能动这三行；两种结局都 exit 0（main 不抛异常即 0）。
|
||
预算 stop 是第三种结局：打印 "goal stopped after rounds: ..."
|
||
且任务记 interrupted（明天预算重置后可续跑），不冒充 unmet。
|
||
回合异常是第四种结局：turn 或验收抛错时不向上传播（否则
|
||
进程非 0 退出、无结论行，云端旧语义会判 terminal failed，
|
||
断点续跑活路就断了），而是记 interrupted、打印
|
||
"goal interrupted after rounds: <原因>"，exit 0，任务可
|
||
/goal-resume 断点续跑（2026-10-28 长 goal 实弹：宿主机
|
||
内存挤兑致回合挂起 59 分钟、恢复时异常退出，旧语义直接
|
||
判 failed，60 分钟工作无续跑口）。
|
||
goal-unmet 穷尽轮次后不允许 goal-resume（按既有规则须新建
|
||
任务），goal-resume 只接受 goal-running / interrupted / running。
|
||
"""
|
||
task["goal"] = goal_text
|
||
task["verify_command"] = verify
|
||
task["goal_max_rounds"] = int(max_rounds)
|
||
task["goal_rounds_done"] = int(start_round) - 1
|
||
task["goal_last_verify"] = last_verify
|
||
# 循环前先落盘一次：否则第一轮 turn 进行中被杀时，磁盘上的
|
||
# 任务还没有 goal/verify_command，goal-resume 会因“没有记录”
|
||
# 而拒绝——断点续跑必须在第一轮开始前就已可续跑。
|
||
task["status"] = "goal-running"
|
||
save_state(state_path, state)
|
||
ok = False
|
||
budget_stopped = False
|
||
goal_usage = None
|
||
|
||
def _interrupt_loop(exc):
|
||
# 回合异常的统一出口：不断言、不向上传播，记 interrupted
|
||
# 并打印标记行，云端据此标 interrupted（可断点续跑）。
|
||
# 再落盘一次已尽力（二次异常直接吞掉，标记行照常打印；
|
||
# 即使落盘也失败，云端无结论 + 非 0 退出时仍有
|
||
# interrupted 兜底，正常路径下本函数返回后 main exit 0）。
|
||
task["status"] = "interrupted"
|
||
task["goal_last_turn_status"] = "interrupted"
|
||
task["goal_interrupted_reason"] = str(exc).replace(
|
||
"\n", " ")[:200]
|
||
try:
|
||
save_state(state_path, state)
|
||
except Exception:
|
||
pass
|
||
print("goal interrupted after rounds: %s"
|
||
% str(exc).replace("\n", " ")[:500])
|
||
return {"ok": False, "budget_stopped": False,
|
||
"interrupted": True,
|
||
"rounds_done": task["goal_rounds_done"],
|
||
"last_verify": last_verify, "usage": goal_usage}
|
||
|
||
# max_rounds=0 是不限轮次：while 循环只在模型宣告、预算耗尽
|
||
# 或取消时结束；非 0 时按旧语义穷尽轮次即 unmet。
|
||
round_no = int(start_round) - 1
|
||
# 连续失败轮次护栏：回合瞬间失败（如密钥缺失）时不限轮次
|
||
# 循环会无限空转（2026-10-29 实弹：断点后 274 轮全失败，
|
||
# state 文件持续增长）。3 轮连续失败即走中断出口，任务
|
||
# 记 interrupted、可断点续跑，不再原地空转；偶发单轮失败
|
||
# 不受影响（计数在成功轮清零）。
|
||
consecutive_failed = 0
|
||
while True:
|
||
round_no += 1
|
||
if int(max_rounds) != 0 and round_no > int(max_rounds):
|
||
break
|
||
if cfg is not None and budget_exceeded(state, cfg):
|
||
budget_stopped = True
|
||
print(f"goal stopped: 预算已用完（max_daily_turns），"
|
||
f"第 {round_no} 轮未开始，任务可续跑")
|
||
break
|
||
if round_no == 1 and not last_verify and not last_turn_status:
|
||
prompt = goal_text
|
||
if verify is None:
|
||
# 新形态首轮必须把标记约定写进提示，否则模型不知
|
||
# 道怎么宣告完成，循环永远等不到结束信号。
|
||
prompt = (f"{goal_text}\n\n"
|
||
f"完成方式：这是目标循环，你可以多轮工作；"
|
||
f"当你确认目标已完全达成时，在回复末尾单独一行输出："
|
||
f"{GOAL_DONE_MARKER}。未达成时不要输出这一行。")
|
||
elif last_turn_status in ("timeout", "interrupted"):
|
||
# 上轮回合根本没跑完：说清是回合中断，不是验收失败，
|
||
# 否则 agent 会对着验收输出修一个不存在的问题。
|
||
prompt = (f"上一轮回合未完成（状态：{last_turn_status}），"
|
||
f"请继续完成目标，不要重复已完成的部分。\n"
|
||
f"目标：{goal_text}\n")
|
||
if verify is not None:
|
||
prompt += (f"验收命令：{verify}\n"
|
||
f"上一轮验收输出：\n{last_verify}\n")
|
||
else:
|
||
prompt += (f"确认目标已完全达成时，在回复末尾单独一行输出："
|
||
f"{GOAL_DONE_MARKER}。\n")
|
||
prompt += f"上一轮部分文本：\n{last_turn_text[:1000]}"
|
||
elif verify is None:
|
||
# 新形态续轮：同一 thread 已有历史，只需说明“还没
|
||
# 宣告达成、继续做”，并重申标记约定。
|
||
prompt = (f"上一轮未宣告目标达成，请继续完成目标，"
|
||
f"不要重复已完成的部分。\n"
|
||
f"目标：{goal_text}\n"
|
||
f"确认目标已完全达成时，在回复末尾单独一行输出："
|
||
f"{GOAL_DONE_MARKER}。")
|
||
else:
|
||
# 生产教训：只说“验收未通过” agent 不知道改什么，
|
||
# 2026-10-25 实测 prod-retry 3 轮都原地不动。
|
||
# 必须把验收命令和失败输出喂回去，才是可用的目标循环。
|
||
prompt = (f"上一轮验收未通过，请针对性修复，不要重复已完成的部分。\n"
|
||
f"目标：{goal_text}\n"
|
||
f"验收命令：{verify}\n"
|
||
f"上一轮验收输出：\n{last_verify}")
|
||
try:
|
||
result = run_task_turn(server, state, task, prompt,
|
||
model, effort, sandbox, timeout=timeout,
|
||
interrupt_file=_interrupt_request_path(
|
||
state_path, task["name"])
|
||
if task.get("name") else None)
|
||
except Exception as exc:
|
||
return _interrupt_loop(exc)
|
||
goal_usage = _add_usage(goal_usage, result.get("usage"))
|
||
last_turn_status = result["status"]
|
||
last_turn_text = result.get("text", "") or ""
|
||
if last_turn_status == "failed":
|
||
consecutive_failed += 1
|
||
if consecutive_failed >= 3:
|
||
return _interrupt_loop(
|
||
Exception("连续 3 轮回合失败，停止空转"))
|
||
else:
|
||
consecutive_failed = 0
|
||
if verify is None:
|
||
# 新形态没有验收命令可跑，v_rc 记 -（轮次行格式不变，
|
||
# 云端解析只读 verify_out 做 unmet 详情，不受影响）。
|
||
v_rc, v_out = None, ""
|
||
else:
|
||
try:
|
||
v_rc, v_out = None, None
|
||
if verify_timeout and verify_timeout > 0:
|
||
v = subprocess.run(verify, shell=True, cwd=task["cwd"],
|
||
capture_output=True, text=True,
|
||
timeout=verify_timeout)
|
||
else:
|
||
v = subprocess.run(verify, shell=True, cwd=task["cwd"],
|
||
capture_output=True, text=True)
|
||
v_rc = v.returncode
|
||
v_out = (v.stdout + v.stderr).strip()[:1000] or \
|
||
f"exit code {v.returncode}"
|
||
except subprocess.TimeoutExpired as exc:
|
||
# 验测本身也可能很长（测试套件）：超时是验测失败，
|
||
# 不是进程挂死，喂回下一轮让 agent 缩小验测范围。
|
||
v_rc = 124
|
||
out = (exc.stdout or "") if isinstance(exc.stdout, str) \
|
||
else ""
|
||
err = (exc.stderr or "") if isinstance(exc.stderr, str) \
|
||
else ""
|
||
v_out = (f"verify timeout after {verify_timeout}s: "
|
||
f"{(out + err).strip()[:900]}")
|
||
except Exception as exc:
|
||
# 验收命令本身起不来（OSError 等）不是验收失败，
|
||
# 同样走中断出口，不让异常杀掉整个循环。
|
||
return _interrupt_loop(exc)
|
||
last_verify = v_out
|
||
task["goal_rounds_done"] = round_no
|
||
task["goal_last_verify"] = last_verify
|
||
task["goal_last_turn_status"] = last_turn_status
|
||
# 检查点：非终局轮次先记 goal-running，终局轮次由下面
|
||
# 统一改写；这样被杀后 state 里能看出循环没跑完。
|
||
task["status"] = "goal-running"
|
||
try:
|
||
save_state(state_path, state)
|
||
except Exception as exc:
|
||
return _interrupt_loop(exc)
|
||
print(f"round {round_no}: turn={result['status']} "
|
||
f"verify_rc={v_rc if v_rc is not None else '-'} "
|
||
f"verify_out={last_verify[:300]}")
|
||
if verify is None:
|
||
# 模型自决：只有完整跑完的回合里的标记行才算数。
|
||
if (result["status"] == "completed"
|
||
and _llm_declared_done(last_turn_text)):
|
||
ok = True
|
||
break
|
||
elif v_rc == 0:
|
||
ok = True
|
||
break
|
||
if budget_stopped:
|
||
task["status"] = "interrupted"
|
||
task["goal_stopped_reason"] = "budget exceeded: max_daily_turns"
|
||
# goal 预算中断以前只打印：与 dispatch 前置门同款，
|
||
# 补一次 budget-exceeded 通知（没配命令时只写日志）。
|
||
notify_failure(state_path, cfg, task.get("name") or "goal",
|
||
{"status": "budget-exceeded",
|
||
"text": "budget exceeded: max_daily_turns"})
|
||
else:
|
||
task["status"] = "goal-met" if ok else "goal-unmet"
|
||
if task["status"] == "goal-unmet":
|
||
notify_failure(state_path, cfg, task["name"],
|
||
{"status": "goal-unmet", "text": last_verify[:200]})
|
||
save_state(state_path, state)
|
||
if budget_stopped:
|
||
print("goal stopped after rounds: budget exceeded")
|
||
else:
|
||
print(f"goal {'met' if ok else 'unmet'} after rounds")
|
||
if goal_usage:
|
||
print(f"usage input={goal_usage['input_tokens']} "
|
||
f"cached={goal_usage['cached_input_tokens']} "
|
||
f"cache_write={goal_usage['cache_write_input_tokens']} "
|
||
f"output={goal_usage['output_tokens']} "
|
||
f"reasoning={goal_usage['reasoning_output_tokens']} "
|
||
f"total={goal_usage['total_tokens']} "
|
||
f"completions={goal_usage['completions']}")
|
||
return {"ok": ok, "budget_stopped": budget_stopped,
|
||
"interrupted": False,
|
||
"rounds_done": task["goal_rounds_done"],
|
||
"last_verify": last_verify, "usage": goal_usage}
|
||
|
||
|
||
def _load_config_file(path):
|
||
if path and os.path.exists(path):
|
||
try:
|
||
return json.load(open(path))
|
||
except Exception as e:
|
||
raise RuntimeError(f"配置文件读失败 {path}: {e}")
|
||
return {}
|
||
|
||
|
||
def _effective_cfg(cfg, ignore_host_budget):
|
||
"""G9：--ignore-host-budget 时的生效配置。
|
||
|
||
只清 max_daily_turns 一项（其它配置原样），返回新 dict
|
||
不改原 cfg；不传开关时原样返回。main 用它，单测直接
|
||
测它，不用拉起 host.py 子进程。"""
|
||
if not ignore_host_budget:
|
||
return cfg
|
||
out = dict(cfg or {})
|
||
out["max_daily_turns"] = None
|
||
return out
|
||
|
||
|
||
def _resolve_path(base_dir, p):
|
||
if not p:
|
||
return p
|
||
if os.path.isabs(p):
|
||
return p
|
||
return os.path.join(base_dir, p)
|
||
|
||
|
||
def _thread_cli_result(resp, method):
|
||
# thread-* 子命令的冻结契约：成功只打 result 内容，失败由调用方
|
||
# 打 error JSON 并以非零退出；result 为空时打空对象，保持 stdout
|
||
# 始终是一个 JSON 对象。
|
||
if not resp or "error" in resp:
|
||
raise RuntimeError(f"{method} 失败：{resp}")
|
||
result = resp.get("result")
|
||
return result if result is not None else {}
|
||
|
||
|
||
def main():
|
||
ap = argparse.ArgumentParser()
|
||
ap.add_argument("--codex-home", default=None)
|
||
ap.add_argument("--codex-bin", default=None)
|
||
ap.add_argument("--config", default=None,
|
||
help="项目配置文件，默认 host.py 同目录的 host-config.json（若存在）")
|
||
ap.add_argument("--state", default=None)
|
||
ap.add_argument("--model", default=None)
|
||
ap.add_argument("--model-provider", default=None,
|
||
help="模型 provider 覆盖：sub2api 时以 -c 注入网关"
|
||
" provider（不改 config.toml）；不传则走配置")
|
||
ap.add_argument("--effort", default=None)
|
||
ap.add_argument("--sandbox", default=None)
|
||
ap.add_argument("--approval-policy", default=None)
|
||
ap.add_argument("--approval", default=None, choices=["auto", "deny"])
|
||
ap.add_argument("--turn-timeout", type=int, default=None,
|
||
help="单轮等待上限（秒）；0 表示不限时、对齐 Codex App "
|
||
"等 turn/completed 的语义；不传时用配置 "
|
||
"turn_timeout，未配则默认 600")
|
||
ap.add_argument("--progress-interval", type=float, default=None,
|
||
help="长等待流式进度间隔（秒）；每满一个间隔往 stderr "
|
||
"打一行 turn progress 并往链路文件记一条 "
|
||
"kind=progress；0 表示关闭进度输出；不传时用配置 "
|
||
"progress_interval，未配则默认 30")
|
||
ap.add_argument("--ignore-host-budget", action="store_true",
|
||
help="G9：忽略 host 的 max_daily_turns 预算（云端 "
|
||
"worker 路径默认传它：云端已有 per-user USD "
|
||
"配额 + worker 认领/轮询门，host 单机 turn "
|
||
"数预算再拦一次只会误拦云端已批的任务；"
|
||
"standalone 直接跑 host.py 时不传，预算照常）")
|
||
sub = ap.add_subparsers(dest="cmd", required=True)
|
||
|
||
p = sub.add_parser("init")
|
||
p.add_argument("--repo", default=None)
|
||
p.add_argument("--force", action="store_true")
|
||
sub.add_parser("config")
|
||
p = sub.add_parser("dispatch")
|
||
p.add_argument("--name", required=True)
|
||
p.add_argument("--task", required=True)
|
||
p.add_argument("--repo", default=None)
|
||
p.add_argument("--seed-dir", default=None,
|
||
help="附件暂存目录：派发时拷进 worktree 再跑 turn")
|
||
p.add_argument("--no-worktree", action="store_true")
|
||
p.add_argument("--depends-on", default=None,
|
||
help="逗号分隔的依赖任务名，依赖未完成前不跑")
|
||
p.add_argument("--priority", type=int, default=0)
|
||
p.add_argument("--max-retries", type=int, default=0)
|
||
|
||
sub.add_parser("list")
|
||
p = sub.add_parser("status"); p.add_argument("--task", required=True)
|
||
p = sub.add_parser("review"); p.add_argument("--task", required=True)
|
||
p = sub.add_parser("merge"); p.add_argument("--task", required=True)
|
||
p.add_argument("--verify", default=None,
|
||
help="合并后验测命令；不传时仓库有 tests/ 就默认跑 unittest")
|
||
p.add_argument("--no-verify", action="store_true")
|
||
p = sub.add_parser("review-comment"); p.add_argument("--task", required=True)
|
||
p.add_argument("--file", required=True); p.add_argument("--line", type=int, required=True)
|
||
p.add_argument("--comment", required=True)
|
||
p = sub.add_parser("fix-review"); p.add_argument("--task", required=True)
|
||
sub.add_parser("doctor")
|
||
sub.add_parser("failures")
|
||
sub.add_parser("health")
|
||
sub.add_parser("restore-state")
|
||
p = sub.add_parser("recover")
|
||
p.add_argument("--older-than", type=int, default=3600,
|
||
help="最后回合超过多少秒未更新才判定中断，默认 3600")
|
||
p = sub.add_parser("daemon")
|
||
p.add_argument("--interval", type=int, default=300,
|
||
help="每轮间隔秒数，默认 300")
|
||
p.add_argument("--foreground", action="store_true",
|
||
help="前台跑循环（测试用，不双 fork）")
|
||
sub.add_parser("daemon-status")
|
||
sub.add_parser("daemon-stop")
|
||
p = sub.add_parser("discard"); p.add_argument("--task", required=True)
|
||
p = sub.add_parser("archive"); p.add_argument("--task", required=True)
|
||
p = sub.add_parser("fork"); p.add_argument("--task", required=True); p.add_argument("--name", required=True); p.add_argument("--sandbox", default=None)
|
||
p = sub.add_parser("compact"); p.add_argument("--task", required=True)
|
||
p = sub.add_parser("code-review"); p.add_argument("--task", required=True)
|
||
p.add_argument("--base-branch", default=None,
|
||
help="评审目标：与该分支的差异（baseBranch）")
|
||
p.add_argument("--commit", default=None,
|
||
help="评审目标：指定提交 sha（commit）")
|
||
p.add_argument("--instructions", default=None,
|
||
help="评审目标：自定义评审指令（custom）")
|
||
p = sub.add_parser("thread-list"); p.add_argument("--cwd", default=None); p.add_argument("--archived", action="store_true")
|
||
p = sub.add_parser("thread-read"); p.add_argument("--thread-id", required=True); p.add_argument("--include-turns", action="store_true")
|
||
p = sub.add_parser("thread-rename"); p.add_argument("--thread-id", required=True); p.add_argument("--name", required=True)
|
||
p = sub.add_parser("thread-archive"); p.add_argument("--thread-id", required=True)
|
||
p = sub.add_parser("thread-unarchive"); p.add_argument("--thread-id", required=True)
|
||
p = sub.add_parser("thread-compact"); p.add_argument("--thread-id", required=True)
|
||
p = sub.add_parser("skills-list"); p.add_argument("--cwd", default=None)
|
||
p = sub.add_parser("apps-list")
|
||
p = sub.add_parser("mcp-servers-list")
|
||
p = sub.add_parser("mcp-servers-reload")
|
||
p = sub.add_parser("mcp-server-oauth-login"); p.add_argument("--server-name", required=True); p.add_argument("--scopes", default=None); p.add_argument("--timeout-secs", type=int, default=None)
|
||
p = sub.add_parser("run"); p.add_argument("--task", required=True); p.add_argument("--prompt", required=True)
|
||
p = sub.add_parser("interrupt"); p.add_argument("--task", required=True)
|
||
p = sub.add_parser("delegate")
|
||
p.add_argument("--parent-task", required=True); p.add_argument("--task", required=True)
|
||
p.add_argument("--name", required=True); p.add_argument("--report-to-parent", action="store_true")
|
||
p = sub.add_parser("goal")
|
||
p.add_argument("--name", required=True); p.add_argument("--task", required=True)
|
||
p.add_argument("--repo", default=None)
|
||
p.add_argument("--seed-dir", default=None,
|
||
help="附件暂存目录：派发时拷进 worktree 再跑 turn")
|
||
p.add_argument("--verify", default=None,
|
||
help="验收命令；不传时走模型自决：模型在回合末尾"
|
||
"单独一行输出“目标已达成”即达标")
|
||
p.add_argument("--max-rounds", type=int, default=3,
|
||
help="轮次上限；0 表示不限轮次（模型宣告完成才结束），"
|
||
"默认 3")
|
||
p.add_argument("--verify-timeout", type=int, default=600,
|
||
help="验测命令上限（秒）；0 表示不限时，默认 600")
|
||
p = sub.add_parser("goal-resume")
|
||
p.add_argument("--task", required=True)
|
||
p.add_argument("--goal-text", default=None,
|
||
help="新目标文本：仅开新目标时用——跳过状态与"
|
||
"goal 记录守卫，在旧线程上从第 1 轮开新目标循环"
|
||
"（新语义：无验收、不传 --max-rounds 时不限轮次）。"
|
||
"断点续跑绝不能传它（不传时沿用原 goal+原验收命令），"
|
||
"传了会把原验收命令覆盖成 None")
|
||
p.add_argument("--max-rounds", type=int, default=None,
|
||
help="不传时沿用原任务的 goal_max_rounds；传了则覆盖总轮次")
|
||
p.add_argument("--verify-timeout", type=int, default=600,
|
||
help="验测命令上限（秒）；0 表示不限时，默认 600")
|
||
p = sub.add_parser("parallel"); p.add_argument("--jobs", required=True)
|
||
p = sub.add_parser("automations"); p.add_argument("--add", action="store_true")
|
||
p.add_argument("--name"); p.add_argument("--task"); p.add_argument("--repo")
|
||
p.add_argument("--interval", type=int, default=86400)
|
||
p.add_argument("--cron-line", action="store_true",
|
||
help="只打印 run-due 的 crontab 行，不装机")
|
||
p.add_argument("--install-cron", action="store_true",
|
||
help="把 run-due 写进当前用户 crontab（幂等，已有则不重复）")
|
||
p.add_argument("--cron-every-minutes", type=int, default=30)
|
||
sub.add_parser("run-due")
|
||
sub.add_parser("github-workflow")
|
||
p = sub.add_parser("pr"); p.add_argument("--task", required=True)
|
||
p.add_argument("--repo", default=None)
|
||
p = sub.add_parser("pr-checks"); p.add_argument("--task", required=True)
|
||
p.add_argument("--repo", default=None)
|
||
p = sub.add_parser("pr-merge"); p.add_argument("--task", required=True)
|
||
p.add_argument("--repo", default=None)
|
||
p = sub.add_parser("sleep"); p.add_argument("--seconds", type=float,
|
||
default=5)
|
||
|
||
args = ap.parse_args()
|
||
# 配置解析：CLI > 配置文件 > 内置默认。配置文件里绝不放密钥，
|
||
# API key 只走环境变量（中转站用 XCM_API_KEY 那种），这是硬规则。
|
||
config_path = args.config or os.path.join(HERE, "host-config.json")
|
||
cfg = _load_config_file(config_path) if os.path.exists(config_path) else {}
|
||
cfg_dir = os.path.dirname(os.path.abspath(config_path))
|
||
if args.cmd == "init":
|
||
if os.path.exists(config_path) and not args.force:
|
||
print(f"配置文件已存在：{config_path}，加 --force 才会覆盖")
|
||
return
|
||
init_cfg = {
|
||
"repo": args.repo or cfg.get("repo") or "",
|
||
"model": "gpt-5.6-terra", "effort": "low",
|
||
"sandbox": "workspace-write", "approval_policy": "never",
|
||
"approval": "deny", "codex_home": "",
|
||
"state": "host-state.json",
|
||
"max_daily_turns": None,
|
||
"notify_command": "",
|
||
"_note": "API key 不放这里，只走环境变量；"
|
||
"max_daily_turns 是按 turn 数算的预算（协议不暴露 token 数）；"
|
||
"notify_command 是失败时跑的本地 shell 命令，可用 {task} {status}",
|
||
}
|
||
json.dump(init_cfg, open(config_path, "w"), indent=2,
|
||
ensure_ascii=False)
|
||
print(f"配置文件已写：{config_path}，请按需改 repo/model 后再用")
|
||
return
|
||
# 有效配置（init/config 之外的命令都用它）
|
||
# G9：只清预算这一项，notify_command 等其它配置不动；
|
||
# 本进程内所有 budget_exceeded(state, cfg) 检查随之失效，
|
||
# dispatch 前置门、重试前检查、goal 循环顶检查一并放行。
|
||
cfg = _effective_cfg(cfg, getattr(args, "ignore_host_budget", False))
|
||
args.model = args.model or cfg.get("model") or "gpt-5.6-terra"
|
||
args.effort = args.effort or cfg.get("effort") or "low"
|
||
args.sandbox = args.sandbox or cfg.get("sandbox")
|
||
args.approval_policy = args.approval_policy or cfg.get("approval_policy") or "never"
|
||
args.approval = args.approval or cfg.get("approval") or "deny"
|
||
# turn-timeout 解析：CLI > 配置 > 默认 600；0 保持为 0（不限时），
|
||
# 不能用 or 串联，否则 0 会被当成缺失而吞掉。
|
||
if args.turn_timeout is None:
|
||
args.turn_timeout = cfg.get("turn_timeout")
|
||
if args.turn_timeout is None:
|
||
args.turn_timeout = 600
|
||
# progress-interval 解析：同款 CLI > 配置 > 默认 30；0 保持为 0
|
||
#（关闭进度输出），不能用 or 串联。
|
||
if args.progress_interval is None:
|
||
args.progress_interval = cfg.get("progress_interval")
|
||
if args.progress_interval is None:
|
||
args.progress_interval = 30
|
||
args.codex_home = args.codex_home or _resolve_path(cfg_dir, cfg.get("codex_home"))
|
||
args.codex_bin = args.codex_bin or _resolve_path(cfg_dir, cfg.get("codex_bin"))
|
||
args.state = args.state or _resolve_path(cfg_dir, cfg.get("state")) or DEFAULT_STATE
|
||
if args.cmd == "config":
|
||
print(json.dumps({
|
||
"config_file": config_path if os.path.exists(config_path) else None,
|
||
"model": args.model, "effort": args.effort,
|
||
"sandbox": args.sandbox, "approval_policy": args.approval_policy,
|
||
"approval": args.approval, "codex_home": args.codex_home,
|
||
"codex_bin": args.codex_bin, "state": args.state,
|
||
"repo": cfg.get("repo"),
|
||
}, indent=2, ensure_ascii=False))
|
||
return
|
||
# dispatch/goal/automations 的 repo 缺省用配置里的 repo
|
||
if args.cmd in ("dispatch", "goal"):
|
||
if not args.repo:
|
||
args.repo = cfg.get("repo")
|
||
if not args.repo:
|
||
raise RuntimeError("缺少 --repo，也没有在 host-config.json 里配 repo")
|
||
if args.cmd == "automations" and (args.cron_line or args.install_cron):
|
||
# cron 行生成：只打印 / 装机，不跑模型，不花钱。
|
||
bin_path = args.codex_bin or default_codex_bin()
|
||
log_path = os.path.join(os.path.dirname(os.path.abspath(args.state)),
|
||
"host-run.log")
|
||
line = (f"*/{args.cron_every_minutes} * * * * cd {HERE} && "
|
||
f"CODEX_BIN={bin_path} python3 host.py --config {config_path} "
|
||
f"run-due >> {log_path} 2>&1 # codex-host-run-due")
|
||
if args.cron_line:
|
||
print(line)
|
||
print("注意：这行本身不生效，要装进 crontab 才生效；"
|
||
"VM 会被平台回收，crontab 不一定能活过回收，"
|
||
"常驻机器用 crontab，VM 上更稳的是 Muse cron 定时调 run-due。")
|
||
return
|
||
if args.install_cron:
|
||
cur = subprocess.run(["crontab", "-l"], capture_output=True,
|
||
text=True).stdout or ""
|
||
if "codex-host-run-due" in cur:
|
||
print("crontab 里已有 codex-host-run-due，未重复安装")
|
||
return
|
||
new = (cur.rstrip("\n") + "\n" + line + "\n") if cur.strip() else line + "\n"
|
||
subprocess.run(["crontab", "-"], input=new, text=True, check=True)
|
||
print(f"已安装 crontab：{line}")
|
||
return
|
||
state = load_state(args.state)
|
||
if args.cmd == "doctor":
|
||
print(json.dumps(check_state(args.state, state), indent=2,
|
||
ensure_ascii=False))
|
||
return
|
||
if args.cmd == "failures":
|
||
failed = [n for n, t in state["tasks"].items()
|
||
if t.get("status") in ("failed", "goal-unmet", "merge-failed")]
|
||
print(json.dumps({"failed_tasks": failed}, indent=2, ensure_ascii=False))
|
||
return
|
||
if args.cmd == "review-comment":
|
||
t = state["tasks"][args.task]
|
||
c = add_review_comment(t, args.file, args.line, args.comment)
|
||
save_state(args.state, state)
|
||
print(json.dumps(c, indent=2, ensure_ascii=False))
|
||
return
|
||
if args.cmd == "github-workflow":
|
||
repo = cfg.get("repo")
|
||
if not repo:
|
||
print("缺少 repo：请在配置里配 repo")
|
||
raise SystemExit(1)
|
||
changed, path = generate_github_workflow(repo)
|
||
print(json.dumps({"path": path,
|
||
"changed": changed}, indent=2, ensure_ascii=False))
|
||
return
|
||
if args.cmd in ("pr", "pr-checks", "pr-merge"):
|
||
if args.task not in state["tasks"]:
|
||
print(f"任务不存在：{args.task}")
|
||
raise SystemExit(1)
|
||
t = state["tasks"][args.task]
|
||
repo = args.repo or t.get("repo") or cfg.get("repo")
|
||
try:
|
||
if args.cmd == "pr":
|
||
if not repo:
|
||
print("缺少 repo：任务和配置里都没有 repo")
|
||
raise SystemExit(1)
|
||
info = create_pr(repo, t)
|
||
save_state(args.state, state)
|
||
print(json.dumps(info, indent=2, ensure_ascii=False))
|
||
elif args.cmd == "pr-checks":
|
||
info = get_pr_checks(t, repo=repo)
|
||
print(json.dumps(info, indent=2, ensure_ascii=False))
|
||
else:
|
||
ok, msg = merge_pr(t, repo=repo)
|
||
if ok:
|
||
save_state(args.state, state)
|
||
print(json.dumps({"merged": True, "message": msg},
|
||
indent=2, ensure_ascii=False))
|
||
else:
|
||
print(msg)
|
||
raise SystemExit(1)
|
||
except SystemExit:
|
||
raise
|
||
except Exception as e:
|
||
# 失败时打印错误原文（gh/git 的 stderr 已包含在异常里），
|
||
# 不编造 PR URL，不静默失败。
|
||
print(str(e))
|
||
raise SystemExit(1)
|
||
return
|
||
if args.cmd == "sleep":
|
||
# clock.sleep 的 host 形态：纯等待，不碰任务、不跑模型、
|
||
# 不改 state。0-60 秒（与服务端 /sleep 上限同款），
|
||
# 超限是调用方传错，报错而不是静默钳制；Ctrl-C 打断
|
||
# 时按惯例 exit 130 并报告已睡时长，不谎称睡满。
|
||
seconds = args.seconds
|
||
if seconds is None or seconds < 0 or seconds > 60:
|
||
print("seconds 必须在 0 到 60 之间")
|
||
raise SystemExit(1)
|
||
slept = 0.0
|
||
try:
|
||
while slept < seconds:
|
||
chunk = min(0.5, seconds - slept)
|
||
time.sleep(chunk)
|
||
slept += chunk
|
||
except KeyboardInterrupt:
|
||
print(json.dumps({"slept_seconds": slept,
|
||
"interrupted": True}, ensure_ascii=False))
|
||
raise SystemExit(130)
|
||
print(json.dumps({"slept_seconds": slept,
|
||
"interrupted": False}, ensure_ascii=False))
|
||
return
|
||
if args.cmd == "health":
|
||
report = run_health_check(args.state, cfg, args.codex_bin or default_codex_bin(),
|
||
args.codex_home, args.approval)
|
||
print(json.dumps(report, indent=2, ensure_ascii=False))
|
||
if not report["ok"]:
|
||
raise SystemExit(1)
|
||
return
|
||
if args.cmd == "restore-state":
|
||
ok, msg = restore_state(args.state)
|
||
print(msg)
|
||
if not ok:
|
||
raise SystemExit(1)
|
||
return
|
||
if args.cmd == "recover":
|
||
strict, err = load_state_strict(args.state)
|
||
if err is not None:
|
||
print(f"state 已损坏，recover 不会覆盖它，先跑 restore-state：{err}")
|
||
raise SystemExit(1)
|
||
recovered = recover_interrupted(state, args.state,
|
||
older_than=args.older_than, cfg=cfg)
|
||
if recovered:
|
||
save_state(args.state, state)
|
||
print(json.dumps({"recovered": recovered}, indent=2, ensure_ascii=False))
|
||
return
|
||
if args.cmd == "daemon-status":
|
||
pidfile = os.path.join(os.path.dirname(os.path.abspath(args.state)),
|
||
"host-daemon.pid")
|
||
logfile = os.path.join(os.path.dirname(os.path.abspath(args.state)),
|
||
"host-daemon.log")
|
||
alive = False
|
||
pid = None
|
||
if os.path.exists(pidfile):
|
||
try:
|
||
pid = int(open(pidfile).read().strip())
|
||
os.kill(pid, 0)
|
||
alive = True
|
||
except Exception:
|
||
alive = False
|
||
tail = ""
|
||
if os.path.exists(logfile):
|
||
try:
|
||
tail = "\n".join(open(logfile).read().splitlines()[-5:])
|
||
except Exception:
|
||
pass
|
||
print(json.dumps({"alive": alive, "pid": pid, "pidfile": pidfile,
|
||
"log_tail": tail}, indent=2, ensure_ascii=False))
|
||
return
|
||
if args.cmd == "daemon-stop":
|
||
pidfile = os.path.join(os.path.dirname(os.path.abspath(args.state)),
|
||
"host-daemon.pid")
|
||
if not os.path.exists(pidfile):
|
||
print("daemon 未运行（无 pidfile）")
|
||
return
|
||
try:
|
||
pid = int(open(pidfile).read().strip())
|
||
except Exception:
|
||
print("pidfile 损坏，直接删除")
|
||
os.remove(pidfile)
|
||
return
|
||
try:
|
||
os.kill(pid, 15)
|
||
for _ in range(10):
|
||
time.sleep(0.5)
|
||
try:
|
||
os.kill(pid, 0)
|
||
except Exception:
|
||
break
|
||
else:
|
||
os.kill(pid, 9)
|
||
except Exception:
|
||
pass
|
||
try:
|
||
os.remove(pidfile)
|
||
except Exception:
|
||
pass
|
||
print(f"daemon 已停：pid {pid}")
|
||
return
|
||
if args.cmd == "daemon":
|
||
state_dir = os.path.dirname(os.path.abspath(args.state))
|
||
pidfile = os.path.join(state_dir, "host-daemon.pid")
|
||
logfile = os.path.join(state_dir, "host-daemon.log")
|
||
if not args.foreground:
|
||
if os.path.exists(pidfile):
|
||
try:
|
||
old_pid = int(open(pidfile).read().strip())
|
||
os.kill(old_pid, 0)
|
||
print(f"daemon 已在运行：pid {old_pid}，先 daemon-stop 再起")
|
||
return
|
||
except Exception:
|
||
pass
|
||
if not daemonize(pidfile, logfile):
|
||
print(f"daemon 已在后台启动：pidfile {pidfile}，log {logfile}")
|
||
return
|
||
# 子进程 / 前台：循环跑 recover + run-due，每轮都是独立
|
||
# 子进程调 host.py CLI，这样一轮崩了不带走 daemon 本体，
|
||
# 且 run-due 的模型回合数受自动化本身和预算控制。
|
||
import signal
|
||
stop = {"flag": False}
|
||
|
||
def _on_term(signum, frame):
|
||
stop["flag"] = True
|
||
signal.signal(signal.SIGTERM, _on_term)
|
||
signal.signal(signal.SIGINT, _on_term)
|
||
print(f"host daemon started pid={os.getpid()} interval={args.interval}s "
|
||
f"config={config_path} state={args.state}", flush=True)
|
||
while not stop["flag"]:
|
||
ts = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
|
||
print(f"{ts} daemon tick: recover + run-due", flush=True)
|
||
for subcmd in (["recover", "--older-than", "3600"], ["run-due"]):
|
||
try:
|
||
subprocess.run([sys.executable, os.path.join(HERE, "host.py"),
|
||
"--config", config_path] + subcmd,
|
||
timeout=3600)
|
||
except Exception as e:
|
||
print(f"{ts} daemon subcommand {subcmd} 失败：{e}", flush=True)
|
||
for _ in range(args.interval):
|
||
if stop["flag"]:
|
||
break
|
||
time.sleep(1)
|
||
try:
|
||
os.remove(pidfile)
|
||
except Exception:
|
||
pass
|
||
print("host daemon stopped", flush=True)
|
||
return
|
||
srv = Server(codex_bin=args.codex_bin, codex_home=args.codex_home,
|
||
approval=args.approval,
|
||
progress_interval=args.progress_interval,
|
||
model_provider=args.model_provider)
|
||
# 实时链路：只给跑 turn 的命令设链路文件（dispatch/run/goal/
|
||
# delegate）；只读命令不设。名字映射与 worker 派发一致：
|
||
# dispatch/goal/delegate 用 --name，run 用 --task。
|
||
_trace_name = None
|
||
if args.cmd in ("dispatch", "goal", "delegate"):
|
||
_trace_name = getattr(args, "name", None)
|
||
elif args.cmd == "run":
|
||
_trace_name = getattr(args, "task", None)
|
||
if _trace_name:
|
||
srv.set_trace_file(args.state, _trace_name)
|
||
try:
|
||
srv.initialize()
|
||
if args.cmd == "dispatch":
|
||
if budget_exceeded(state, cfg):
|
||
print("预算已用完（max_daily_turns），本次不派发，见 doctor 的 usage")
|
||
# 前置预算门以前只打印：配了 notify_command 的
|
||
# 用户收不到任何通知。补一次 budget-exceeded
|
||
# 通知；没配命令时 notify_failure 只写日志，
|
||
# 不会无配置刷屏。
|
||
notify_failure(args.state, cfg, args.name,
|
||
{"status": "budget-exceeded",
|
||
"text": "budget exceeded: max_daily_turns"})
|
||
return
|
||
deps = [d.strip() for d in args.depends_on.split(",")
|
||
if d.strip()] if args.depends_on else []
|
||
task = dispatch(srv, state, args.name, args.task, args.repo,
|
||
args.model, args.effort, args.sandbox,
|
||
args.approval_policy,
|
||
use_worktree=not args.no_worktree,
|
||
depends_on=deps, priority=args.priority,
|
||
max_retries=args.max_retries,
|
||
seed_dir=args.seed_dir)
|
||
# 生产教训：先落盘任务登记再跑 turn，中途被杀/超时后
|
||
# 任务还在 host-state 里，可用 run 续跑；否则 turn 中断
|
||
# 就等于任务丢了（2026-10-25 委派评审 300 秒超时实测）。
|
||
save_state(args.state, state)
|
||
result = run_task_turn_with_retry(srv, state, task, args.task,
|
||
args.model, args.effort,
|
||
args.sandbox, cfg=cfg,
|
||
timeout=args.turn_timeout,
|
||
interrupt_file=_interrupt_request_path(args.state, args.name))
|
||
if result["status"] != "completed":
|
||
notify_failure(args.state, cfg, args.name, result)
|
||
save_state(args.state, state)
|
||
print(json.dumps({"task": args.name, "thread_id": task["thread_id"],
|
||
"cwd": task["cwd"], "status": task["status"],
|
||
"text": result.get("text", ""),
|
||
"error": result.get("error"),
|
||
"turn_id": result.get("turn_id"),
|
||
"root_turn_id": result.get("root_turn_id"),
|
||
"usage": result.get("usage")},
|
||
indent=2, ensure_ascii=False))
|
||
elif args.cmd == "list":
|
||
for name, t in state["tasks"].items():
|
||
print(f"{name}: thread={t['thread_id'][:8]} status={t['status']} "
|
||
f"cwd={t['cwd']} branch={t.get('branch')}")
|
||
print(f"app-server threads: "
|
||
f"{json.dumps(srv.list_threads(), ensure_ascii=False)[:500]}")
|
||
elif args.cmd == "status":
|
||
t = state["tasks"][args.task]
|
||
print(json.dumps(t, indent=2, ensure_ascii=False))
|
||
elif args.cmd == "review":
|
||
t = state["tasks"][args.task]
|
||
d = task_diff(t["cwd"])
|
||
t["diff"] = d
|
||
save_state(args.state, state)
|
||
print(f"status:\n{d['status']}\nuntracked:\n{d['untracked']}\n"
|
||
f"diff:\n{d['diff'][:8000]}")
|
||
for rel, content in (d.get("untracked_contents") or {}).items():
|
||
print(f"\n--- untracked file: {rel} ---\n{content}")
|
||
oc = open_review_comments(t)
|
||
if oc:
|
||
print("\n--- open review comments ---")
|
||
for c in oc:
|
||
print(f"{c['file']}:{c['line']}: {c['comment']}")
|
||
elif args.cmd == "merge":
|
||
t = state["tasks"][args.task]
|
||
if not t.get("repo") or not t.get("branch"):
|
||
print("非 git worktree 任务，无法 merge，直接看 diff 后手动处理")
|
||
return
|
||
# 合并 + 验测 + 回滚：验测命令默认仓库有 tests/ 就跑
|
||
# unittest，--no-verify 跳过，--verify 可指定命令。
|
||
# 验测失败时 merge_task 已 reset 回合并前 HEAD，分支和
|
||
# worktree 保留，任务标 merge-failed，不删改动。
|
||
verify_cmd = "" if args.no_verify else args.verify
|
||
ok, msg = merge_task(t["repo"], t["cwd"], t["branch"],
|
||
args.task, verify_command=verify_cmd)
|
||
print(msg)
|
||
if ok:
|
||
t["status"] = "merged"
|
||
else:
|
||
t["status"] = "merge-failed"
|
||
notify_failure(args.state, cfg, args.task,
|
||
{"status": "failed", "text": msg[:200]})
|
||
save_state(args.state, state)
|
||
elif args.cmd == "fix-review":
|
||
t = state["tasks"][args.task]
|
||
oc = open_review_comments(t)
|
||
if not oc:
|
||
print("没有未处理的评审评论")
|
||
return
|
||
srv.resume_thread(t["thread_id"])
|
||
result = run_task_turn(srv, state, t, build_fix_prompt(t),
|
||
args.model, args.effort, args.sandbox,
|
||
timeout=args.turn_timeout,
|
||
interrupt_file=_interrupt_request_path(args.state, args.task))
|
||
if result["status"] == "completed":
|
||
for c in oc:
|
||
c["status"] = "resolved"
|
||
c["resolved_at"] = time.time()
|
||
print(f"fix-review completed，{len(oc)} 条评论已标 resolved")
|
||
else:
|
||
notify_failure(args.state, cfg, args.task, result)
|
||
print(f"fix-review failed，评论仍保持 open：{result}")
|
||
save_state(args.state, state)
|
||
elif args.cmd == "discard":
|
||
if args.task not in state["tasks"]:
|
||
print(f"任务不存在：{args.task}（可能从未派发，或 skipped 任务未落盘）")
|
||
return
|
||
t = state["tasks"][args.task]
|
||
if t.get("repo") and t.get("branch"):
|
||
git(["worktree", "remove", "--force", t["cwd"]], t["repo"])
|
||
git(["branch", "-D", t["branch"]], t["repo"])
|
||
t["status"] = "discarded"
|
||
save_state(args.state, state)
|
||
print(f"discarded {args.task}")
|
||
elif args.cmd == "thread-list":
|
||
# 只读命令，不 resume；cwd/archived 只在给了时才进 params，
|
||
# 不传时服务端按自身缺省过滤。
|
||
try:
|
||
params = {}
|
||
if args.cwd:
|
||
params["cwd"] = args.cwd
|
||
if args.archived:
|
||
params["archived"] = True
|
||
resp = srv.request("thread/list", params, timeout=60)
|
||
print(json.dumps(_thread_cli_result(resp, "thread/list"),
|
||
ensure_ascii=False))
|
||
except SystemExit:
|
||
raise
|
||
except Exception as e:
|
||
print(json.dumps({"error": str(e)}, ensure_ascii=False))
|
||
raise SystemExit(1)
|
||
elif args.cmd == "thread-read":
|
||
try:
|
||
srv.resume_thread(args.thread_id)
|
||
resp = srv.request("thread/read",
|
||
{"threadId": args.thread_id,
|
||
"includeTurns": bool(args.include_turns)},
|
||
timeout=60)
|
||
print(json.dumps(_thread_cli_result(resp, "thread/read"),
|
||
ensure_ascii=False))
|
||
except SystemExit:
|
||
raise
|
||
except Exception as e:
|
||
print(json.dumps({"error": str(e)}, ensure_ascii=False))
|
||
raise SystemExit(1)
|
||
elif args.cmd == "thread-rename":
|
||
try:
|
||
srv.resume_thread(args.thread_id)
|
||
resp = srv.request("thread/name/set",
|
||
{"threadId": args.thread_id,
|
||
"name": args.name},
|
||
timeout=60)
|
||
print(json.dumps(_thread_cli_result(resp, "thread/name/set"),
|
||
ensure_ascii=False))
|
||
except SystemExit:
|
||
raise
|
||
except Exception as e:
|
||
print(json.dumps({"error": str(e)}, ensure_ascii=False))
|
||
raise SystemExit(1)
|
||
elif args.cmd == "thread-archive":
|
||
# 沿用现有 archive 行为：不 resume，直接调 thread/archive。
|
||
try:
|
||
resp = srv.archive_thread(args.thread_id)
|
||
print(json.dumps(_thread_cli_result(resp, "thread/archive"),
|
||
ensure_ascii=False))
|
||
except SystemExit:
|
||
raise
|
||
except Exception as e:
|
||
print(json.dumps({"error": str(e)}, ensure_ascii=False))
|
||
raise SystemExit(1)
|
||
elif args.cmd == "thread-unarchive":
|
||
# 与 archive 一致：不 resume，直接调 thread/unarchive。
|
||
# 已归档 thread 先 resume 会失败（thread/resume 报错 →
|
||
# CLI 退出 1 → 服务端回 502），而 unarchive 本身只认
|
||
# threadId，不需要先恢复会话。
|
||
try:
|
||
resp = srv.request("thread/unarchive",
|
||
{"threadId": args.thread_id},
|
||
timeout=60)
|
||
print(json.dumps(_thread_cli_result(resp, "thread/unarchive"),
|
||
ensure_ascii=False))
|
||
except SystemExit:
|
||
raise
|
||
except Exception as e:
|
||
print(json.dumps({"error": str(e)}, ensure_ascii=False))
|
||
raise SystemExit(1)
|
||
elif args.cmd == "thread-compact":
|
||
try:
|
||
srv.resume_thread(args.thread_id)
|
||
resp = srv.request("thread/compact/start",
|
||
{"threadId": args.thread_id},
|
||
timeout=120)
|
||
print(json.dumps(_thread_cli_result(resp, "thread/compact/start"),
|
||
ensure_ascii=False))
|
||
except SystemExit:
|
||
raise
|
||
except Exception as e:
|
||
print(json.dumps({"error": str(e)}, ensure_ascii=False))
|
||
raise SystemExit(1)
|
||
elif args.cmd == "skills-list":
|
||
# skills/mcp 管理面与 thread-* 子命令同款冻结契约：成功
|
||
# 只打 result 内容，失败打 error JSON 并以非零退出。
|
||
# skills/list 等方法不带 threadId，不需要 resume。
|
||
try:
|
||
params = {}
|
||
if args.cwd:
|
||
params["cwds"] = [args.cwd]
|
||
resp = srv.request("skills/list", params, timeout=60)
|
||
print(json.dumps(_thread_cli_result(resp, "skills/list"),
|
||
ensure_ascii=False))
|
||
except SystemExit:
|
||
raise
|
||
except Exception as e:
|
||
print(json.dumps({"error": str(e)}, ensure_ascii=False))
|
||
raise SystemExit(1)
|
||
elif args.cmd == "apps-list":
|
||
try:
|
||
resp = srv.request("app/list", {}, timeout=60)
|
||
print(json.dumps(_thread_cli_result(resp, "app/list"),
|
||
ensure_ascii=False))
|
||
except SystemExit:
|
||
raise
|
||
except Exception as e:
|
||
print(json.dumps({"error": str(e)}, ensure_ascii=False))
|
||
raise SystemExit(1)
|
||
elif args.cmd == "mcp-servers-list":
|
||
try:
|
||
resp = srv.request("mcpServerStatus/list", {}, timeout=60)
|
||
print(json.dumps(_thread_cli_result(resp, "mcpServerStatus/list"),
|
||
ensure_ascii=False))
|
||
except SystemExit:
|
||
raise
|
||
except Exception as e:
|
||
print(json.dumps({"error": str(e)}, ensure_ascii=False))
|
||
raise SystemExit(1)
|
||
elif args.cmd == "mcp-servers-reload":
|
||
try:
|
||
resp = srv.request("config/mcpServer/reload", None, timeout=60)
|
||
print(json.dumps(_thread_cli_result(resp, "config/mcpServer/reload"),
|
||
ensure_ascii=False))
|
||
except SystemExit:
|
||
raise
|
||
except Exception as e:
|
||
print(json.dumps({"error": str(e)}, ensure_ascii=False))
|
||
raise SystemExit(1)
|
||
elif args.cmd == "mcp-server-oauth-login":
|
||
try:
|
||
params = {"name": args.server_name}
|
||
if args.scopes:
|
||
params["scopes"] = [s for s in args.scopes.split(",") if s]
|
||
if args.timeout_secs is not None:
|
||
params["timeoutSecs"] = args.timeout_secs
|
||
resp = srv.request("mcpServer/oauth/login", params, timeout=120)
|
||
print(json.dumps(_thread_cli_result(resp, "mcpServer/oauth/login"),
|
||
ensure_ascii=False))
|
||
except SystemExit:
|
||
raise
|
||
except Exception as e:
|
||
print(json.dumps({"error": str(e)}, ensure_ascii=False))
|
||
raise SystemExit(1)
|
||
elif args.cmd == "compact":
|
||
t = state["tasks"][args.task]
|
||
# 注意：compact 要求 thread 已在当前 server 里加载，
|
||
# 直接对未加载的旧 thread 调会报 thread not found，
|
||
# 所以先 resume 再 compact，这是协议层面的先后顺序。
|
||
srv.resume_thread(t["thread_id"])
|
||
resp = srv.request("thread/compact/start",
|
||
{"threadId": t["thread_id"]}, timeout=120)
|
||
print(f"compact {args.task}: {resp}")
|
||
elif args.cmd == "code-review":
|
||
# 注意命名：host.py 已有另一条线的 review 子命令
|
||
# （PR 评审评论），本命令换名 code-review，不覆盖它。
|
||
# 目标优先级：commit > instructions > base-branch >
|
||
# 缺省 uncommittedChanges；与 cloud/review.py 的
|
||
# build_review_target 同款组合，非法组合直接报错。
|
||
# compact 同款原因先 resume 再 review（未加载的旧
|
||
# thread 直接 review 会报 thread not found）。
|
||
# app-server 返回 error 时只打印失败并以非零退出，
|
||
# 绝不把失败结果当成评审结论。
|
||
t = state["tasks"][args.task]
|
||
if args.commit:
|
||
target = {"type": "commit", "sha": args.commit}
|
||
elif args.instructions:
|
||
target = {"type": "custom",
|
||
"instructions": args.instructions}
|
||
elif args.base_branch:
|
||
target = {"type": "baseBranch",
|
||
"branch": args.base_branch}
|
||
else:
|
||
target = {"type": "uncommittedChanges"}
|
||
try:
|
||
srv.resume_thread(t["thread_id"])
|
||
result = srv.start_review(t["thread_id"], target)
|
||
except Exception as exc:
|
||
print(f"code-review failed: {exc}")
|
||
raise SystemExit(1)
|
||
print(f"code-review {args.task}: {result}")
|
||
elif args.cmd == "archive":
|
||
t = state["tasks"][args.task]
|
||
srv.archive_thread(t["thread_id"])
|
||
t["status"] = "archived"
|
||
save_state(args.state, state)
|
||
print(f"archived {args.task}")
|
||
elif args.cmd == "fork":
|
||
t = state["tasks"][args.task]
|
||
# 生产教训：fork 只分叉 thread 历史，不自动隔离工作区。
|
||
# 旧实现复用原 worktree，合并后 worktree 已删时 fork 出来的
|
||
# thread 连 cwd 都没了，只能只读。正确做法：有 repo 时给
|
||
# fork 单独建一个新 worktree，并把 cwd 传给 thread/fork。
|
||
new_cwd = t["cwd"]
|
||
new_branch = t.get("branch")
|
||
if t.get("repo"):
|
||
old_cwd = t["cwd"]
|
||
new_cwd, new_branch = make_worktree(t["repo"], args.name)
|
||
if os.path.isdir(old_cwd):
|
||
# 未合并任务：新 worktree 只有主仓内容，没有原任务的
|
||
# 改动。把原 worktree 的文件（不含 .git）拷过去，
|
||
# fork 出来的 thread 历史和工作区才对得上。
|
||
import shutil
|
||
for root, dirs, files in os.walk(old_cwd):
|
||
dirs[:] = [d for d in dirs
|
||
if d not in (".git", "__pycache__")]
|
||
for fn in files:
|
||
if fn.endswith(".pyc") or fn == ".git":
|
||
# .git 在 worktree 里是一个指向 gitdir 的文件，
|
||
# 拷过去会把新 worktree 的指针指回旧 worktree，
|
||
# 导致 git worktree 校验失败（2026-10-25 实测）。
|
||
continue
|
||
src = os.path.join(root, fn)
|
||
rel = os.path.relpath(src, old_cwd)
|
||
dst = os.path.join(new_cwd, rel)
|
||
os.makedirs(os.path.dirname(dst), exist_ok=True)
|
||
shutil.copy2(src, dst)
|
||
# fork 不传 sandbox 时服务端默认 read-only（实测），
|
||
# 所以必须显式传原任务的沙箱，旧任务没存沙箱时默认 workspace-write。
|
||
fork_sandbox = args.sandbox or t.get("sandbox") or "workspace-write"
|
||
new_id = srv.fork_thread(t["thread_id"], cwd=new_cwd,
|
||
sandbox=fork_sandbox)
|
||
nt = dict(t)
|
||
nt.update({"name": args.name, "thread_id": new_id,
|
||
"cwd": new_cwd, "branch": new_branch,
|
||
"status": "forked", "parent_task": args.task})
|
||
state["tasks"][args.name] = nt
|
||
save_state(args.state, state)
|
||
print(f"forked {args.task} -> {args.name} thread={new_id} "
|
||
f"cwd={new_cwd}")
|
||
elif args.cmd == "run":
|
||
t = state["tasks"][args.task]
|
||
srv.resume_thread(t["thread_id"])
|
||
result = run_task_turn_with_retry(srv, state, t, args.prompt,
|
||
args.model, args.effort,
|
||
args.sandbox, cfg=cfg,
|
||
timeout=args.turn_timeout,
|
||
interrupt_file=_interrupt_request_path(args.state, args.task))
|
||
if result["status"] != "completed":
|
||
notify_failure(args.state, cfg, args.task, result)
|
||
save_state(args.state, state)
|
||
print(json.dumps(result, indent=2, ensure_ascii=False))
|
||
elif args.cmd == "interrupt":
|
||
t = state["tasks"][args.task]
|
||
turn_id = None
|
||
for turn in reversed(t.get("turns", [])):
|
||
turn_id = (turn.get("result") or {}).get("turn_id")
|
||
if turn_id:
|
||
break
|
||
if not turn_id:
|
||
print(f"interrupt {args.task}: 没有可中断的 turn_id")
|
||
else:
|
||
srv.resume_thread(t["thread_id"])
|
||
resp = srv.interrupt_turn(t["thread_id"], turn_id)
|
||
# 直接中断生效（快路径）时沿用老行为：只有服务端
|
||
# 接受中断才改状态；它说“没有活动回合”时说明该轮
|
||
# 已结束，硬标 interrupted 会掩盖真实终态。
|
||
if resp and "error" not in resp:
|
||
t["status"] = "interrupted"
|
||
save_state(args.state, state)
|
||
req = _interrupt_request_path(args.state, args.task)
|
||
try:
|
||
if os.path.exists(req):
|
||
os.remove(req)
|
||
except Exception:
|
||
pass
|
||
print(f"interrupt {args.task}: turn={turn_id} resp={resp}")
|
||
else:
|
||
# 直接中断不生效（典型是另一进程在等这个 turn，
|
||
# 本进程的 server 并不拥有它）：写中断请求文件，
|
||
# 属主进程在等 turn 时每秒轮询到后用自己的
|
||
# live 连接发 turn/interrupt 并记 interrupted。
|
||
# 写完后最多等 10 秒看属主是否已响应，超时也
|
||
# 不删文件——属主下次轮询时仍会处理，调用方
|
||
# 也可 Ctrl-C 属主进程 + recover 兜底。
|
||
req = _interrupt_request_path(args.state, args.task)
|
||
try:
|
||
os.makedirs(os.path.dirname(req), exist_ok=True)
|
||
with open(req, "w", encoding="utf-8") as f:
|
||
json.dump({"task": args.task,
|
||
"thread_id": t["thread_id"],
|
||
"turn_id": turn_id,
|
||
"requested_at": time.time()},
|
||
f, ensure_ascii=False)
|
||
except Exception as exc:
|
||
print(f"interrupt {args.task}: 写中断请求失败：{exc}")
|
||
return
|
||
print(f"interrupt {args.task}: 直接中断未生效 "
|
||
f"(resp={resp})，已写中断请求 {req}，"
|
||
f"等待属主进程处理")
|
||
for _ in range(10):
|
||
time.sleep(1)
|
||
try:
|
||
cur = json.load(open(args.state,
|
||
encoding="utf-8"))
|
||
if (cur.get("tasks", {}).get(args.task, {})
|
||
.get("status") == "interrupted"):
|
||
print(f"interrupt {args.task}: 属主已响应，"
|
||
f"任务已记 interrupted")
|
||
break
|
||
except Exception:
|
||
pass
|
||
else:
|
||
print(f"interrupt {args.task}: 属主 10 秒内未响应，"
|
||
f"请求文件仍在 {req}，属主下次轮询时会处理")
|
||
elif args.cmd == "delegate":
|
||
parent = state["tasks"][args.parent_task]
|
||
# 子 thread：cwd 与父任务相同 worktree 时只读更安全，
|
||
# 这里默认沿用父 cwd + read-only，避免子任务改坏父任务的改动
|
||
sub_thread = srv.create_thread(parent["cwd"], model=args.model,
|
||
sandbox="read-only",
|
||
approval_policy="never")
|
||
sub = {"name": args.name, "thread_id": sub_thread,
|
||
"cwd": parent["cwd"], "repo": parent.get("repo"),
|
||
"branch": None, "sandbox": "read-only", "status": "running",
|
||
"prompt": args.task, "parent_task": args.parent_task,
|
||
"created_at": time.time(), "turns": []}
|
||
state["tasks"][args.name] = sub
|
||
save_state(args.state, state)
|
||
# 子代理链路：向父任务链路文件报到（开始 + 结束各一条），
|
||
# 服务端凭标记把子链路文件拼进父任务的代理链；子任务自己
|
||
# 的过程条目仍写它自己的链路文件（main 起动时按 --name 定）。
|
||
child_trace = (os.path.basename(srv.trace_path)
|
||
if srv.trace_path else None)
|
||
_append_trace_marker(
|
||
args.state, args.parent_task,
|
||
{"kind": "agent-start", "id": "agent:" + args.name,
|
||
"name": args.name, "prompt": (args.task or "")[:300],
|
||
"trace": child_trace, "ts": int(time.time() * 1000)})
|
||
result = run_task_turn(srv, state, sub, args.task,
|
||
args.model, args.effort, args.sandbox,
|
||
timeout=args.turn_timeout,
|
||
interrupt_file=_interrupt_request_path(args.state, args.name))
|
||
save_state(args.state, state)
|
||
_append_trace_marker(
|
||
args.state, args.parent_task,
|
||
{"kind": "agent-done", "id": "agent:" + args.name + ":done",
|
||
"name": args.name, "status": sub["status"],
|
||
"trace": child_trace, "ts": int(time.time() * 1000)})
|
||
print(json.dumps({"sub_task": args.name, "status": sub["status"],
|
||
"text": result.get("text", "")},
|
||
indent=2, ensure_ascii=False))
|
||
if args.report_to_parent:
|
||
srv.resume_thread(parent["thread_id"])
|
||
rep = run_task_turn(srv, state, parent,
|
||
f"子任务 {args.name} 的结论如下，请汇总到主任务结论里，"
|
||
f"不要重复做子任务的工作：\n{result.get('text', '')}",
|
||
args.model, args.effort, args.sandbox,
|
||
timeout=args.turn_timeout,
|
||
interrupt_file=_interrupt_request_path(args.state, args.parent_task))
|
||
save_state(args.state, state)
|
||
print(f"reported to parent: {rep.get('text', '')[:1000]}")
|
||
elif args.cmd == "goal":
|
||
task = dispatch(srv, state, args.name, args.task, args.repo,
|
||
args.model, args.effort, args.sandbox,
|
||
args.approval_policy,
|
||
seed_dir=args.seed_dir)
|
||
save_state(args.state, state)
|
||
run_goal_loop(srv, state, args.state, task, args.task,
|
||
args.verify, args.max_rounds, args.model,
|
||
args.effort, args.sandbox, cfg=cfg,
|
||
timeout=args.turn_timeout,
|
||
verify_timeout=args.verify_timeout)
|
||
elif args.cmd == "goal-resume":
|
||
t = state["tasks"][args.task]
|
||
# P1 goal-resume 契约（与 cloud/sandbox.py、cloud/worker.py
|
||
# 三处一致，绝不能混用）：
|
||
# - 开新目标：调用方传 --goal-text 时走下面分支，在旧线程
|
||
# 上从第 1 轮开新循环（verify=None，新语义）。这个分支
|
||
# 只能由“开新目标”的调用方用，断点续跑绝不能传 --goal-text
|
||
# ——传了就会把原 verify_command 覆盖成 None。
|
||
# - 断点续跑：调用方不传 --goal-text 时走下面守卫分支，
|
||
# 沿用原 goal + 原 verify_command + 原 goal_max_rounds，
|
||
# 从 goal_rounds_done 的下一轮续跑，verify 原样传回
|
||
# run_goal_loop（run_goal_loop 会把它再写回同一值，
|
||
# 不是覆盖成 None）。
|
||
if args.goal_text:
|
||
# 旧任务接续 goal 模式（云端 /continue 带 goal_mode=1
|
||
# 时走这里）：旧任务已终局、没有 goal 记录是正常情况，
|
||
# 不能按断点续跑的守卫拒绝。只要线程还在，就在旧线程
|
||
# 上从第 1 轮开新目标循环（新语义：无验收、不传轮次
|
||
# 时不限轮次，模型宣告完成才结束，上界是云端配额）。
|
||
if not t.get("thread_id"):
|
||
print(f"goal-resume 拒绝：{args.task} 没有会话线程，"
|
||
f"不能接续")
|
||
return
|
||
max_rounds = args.max_rounds \
|
||
if args.max_rounds is not None else 0
|
||
srv.resume_thread(t["thread_id"])
|
||
run_goal_loop(srv, state, args.state, t, args.goal_text,
|
||
None, max_rounds,
|
||
args.model, args.effort, args.sandbox, cfg=cfg,
|
||
timeout=args.turn_timeout,
|
||
verify_timeout=args.verify_timeout,
|
||
start_round=1,
|
||
last_verify="",
|
||
last_turn_status=None,
|
||
last_turn_text="")
|
||
return
|
||
if t.get("status") not in ("goal-running", "interrupted",
|
||
"running"):
|
||
print(f"goal-resume 拒绝：{args.task} 当前状态是 "
|
||
f"{t.get('status')}，已终局或已达标；"
|
||
f"goal-unmet / goal-met 后请新建任务，不要同任务续跑")
|
||
return
|
||
if not t.get("goal"):
|
||
print(f"goal-resume 拒绝：{args.task} 没有 goal 记录")
|
||
return
|
||
# 新形态 verify_command 为空、goal_max_rounds 为 0（不限轮次）：
|
||
# 0 是合法值不能用 or 兜底，否则续跑会被悄悄改成 3 轮上限。
|
||
if args.max_rounds is not None:
|
||
max_rounds = args.max_rounds
|
||
elif t.get("goal_max_rounds") is not None:
|
||
max_rounds = t["goal_max_rounds"]
|
||
else:
|
||
max_rounds = 3
|
||
# P1：断点续跑时旧验收形态的 0 轮次（脏数据，0 在旧形态
|
||
# 里表示不限轮次）不按不限轮次跑，钳制到 100 轮（与云端
|
||
# GOAL_MAX_ROUNDS_CAP 同值）；新形态（verify 为空）的 0
|
||
# 保持不限轮次，上界是云端配额。
|
||
if max_rounds == 0 and t.get("verify_command"):
|
||
max_rounds = 100
|
||
srv.resume_thread(t["thread_id"])
|
||
last_turn = (t.get("turns") or [{}])[-1]
|
||
last_result = last_turn.get("result") or {}
|
||
run_goal_loop(srv, state, args.state, t, t["goal"],
|
||
t["verify_command"], max_rounds,
|
||
args.model, args.effort, args.sandbox, cfg=cfg,
|
||
timeout=args.turn_timeout,
|
||
verify_timeout=args.verify_timeout,
|
||
start_round=int(t.get("goal_rounds_done") or 0) + 1,
|
||
last_verify=t.get("goal_last_verify") or "",
|
||
last_turn_status=t.get("goal_last_turn_status"),
|
||
last_turn_text=last_result.get("text", "") or "")
|
||
elif args.cmd == "parallel":
|
||
jobs = json.load(open(args.jobs))
|
||
# 依赖 + 优先级：先拓扑排序，缺依赖/成环直接报错不跑；
|
||
# 按波次派发 + 跑：依赖都 completed 的任务才进本波，
|
||
# 依赖 failed/skipped 的任务标 skipped 不跑，避免在坏
|
||
# 基础上浪费模型调用。关键：依赖任务不是从 HEAD 现建
|
||
# 的——本波派发时从依赖分支建 worktree（见
|
||
# dispatch_from_branch），这样依赖任务的分支是依赖分支
|
||
# 的后代，合并时先合依赖、再合依赖任务才不会重复加同一段。
|
||
ordered = topo_sort_jobs(jobs)
|
||
tasks = {}
|
||
remaining = list(ordered)
|
||
while remaining:
|
||
# 先处理被依赖失败卡住的任务（它们还没派发，
|
||
# 所以这里用一个占位状态记录 skipped）
|
||
blocked = []
|
||
for j in remaining:
|
||
deps = j.get("depends_on") or []
|
||
if deps and any(d in tasks and tasks[d][1]["status"]
|
||
in ("failed", "skipped") for d in deps):
|
||
blocked.append(j)
|
||
for j in blocked:
|
||
print(f"{j['name']}: skipped (依赖失败)")
|
||
skipped_task = {"name": j["name"], "status": "skipped",
|
||
"cwd": None, "thread_id": "",
|
||
"branch": None, "repo": j.get("repo"),
|
||
"depends_on": j.get("depends_on"),
|
||
"priority": j.get("priority", 0),
|
||
"prompt": j["task"], "turns": []}
|
||
state["tasks"][j["name"]] = skipped_task
|
||
tasks[j["name"]] = (j, skipped_task)
|
||
remaining.remove(j)
|
||
if blocked:
|
||
save_state(args.state, state)
|
||
ready = [j for j in remaining
|
||
if all(d in tasks and tasks[d][1]["status"] == "completed"
|
||
for d in (j.get("depends_on") or []))]
|
||
if not ready:
|
||
if remaining:
|
||
print(f"剩余任务无法推进（依赖未完成）："
|
||
f"{[j['name'] for j in remaining]}")
|
||
break
|
||
ready.sort(key=lambda j: (-(j.get("priority") or 0), j["name"]))
|
||
# 本波派发：有依赖且依赖是 git 任务时，从依赖分支
|
||
# 建 worktree（有血缘关系，合并时不重复加同一段）；
|
||
# 否则退化为从 HEAD 建 + 文件拷贝（旧方法，仅兜底）。
|
||
for job in ready:
|
||
deps = job.get("depends_on") or []
|
||
t = None
|
||
if deps and job.get("repo"):
|
||
dep_task = tasks[deps[0]][1]
|
||
if dep_task.get("branch") and dep_task.get("cwd") \
|
||
and os.path.isdir(dep_task["cwd"]):
|
||
t, err = dispatch_from_branch(
|
||
srv, state, job["name"], job["task"],
|
||
job.get("repo", ""), dep_task["branch"],
|
||
dep_task["cwd"], args.model, args.effort,
|
||
job.get("sandbox", args.sandbox),
|
||
args.approval_policy,
|
||
depends_on=deps,
|
||
priority=job.get("priority", 0),
|
||
max_retries=job.get("max_retries", 0))
|
||
if t is None:
|
||
print(f"{job['name']}: 从依赖分支派发失败 "
|
||
f"({err})，退化为 HEAD + 文件拷贝")
|
||
if t is None:
|
||
t = dispatch(srv, state, job["name"], job["task"],
|
||
job.get("repo", ""), args.model, args.effort,
|
||
job.get("sandbox", args.sandbox),
|
||
args.approval_policy,
|
||
use_worktree=bool(job.get("repo")),
|
||
depends_on=deps,
|
||
priority=job.get("priority", 0),
|
||
max_retries=job.get("max_retries", 0))
|
||
if deps and job.get("repo") and tasks[deps[0]][1].get("cwd"):
|
||
copy_tree_excluding_git(tasks[deps[0]][1]["cwd"],
|
||
t["cwd"])
|
||
tasks[job["name"]] = (job, t)
|
||
save_state(args.state, state)
|
||
|
||
def _run(job_task):
|
||
job, t = job_task
|
||
return job["name"], run_task_turn_with_retry(
|
||
srv, state, t, job["task"], args.model,
|
||
args.effort, args.sandbox, cfg=cfg,
|
||
timeout=args.turn_timeout,
|
||
interrupt_file=_interrupt_request_path(args.state, job["name"]))
|
||
with ThreadPoolExecutor(max_workers=len(ready)) as ex:
|
||
for name, result in ex.map(
|
||
_run, [tasks[j["name"]] for j in ready]):
|
||
print(f"{name}: {result['status']} "
|
||
f"{result.get('text', '')[:300]}")
|
||
if result["status"] != "completed":
|
||
notify_failure(args.state, cfg, name, result)
|
||
save_state(args.state, state)
|
||
remaining = [j for j in remaining if j not in ready]
|
||
elif args.cmd == "automations":
|
||
if args.add:
|
||
state["automations"].append({
|
||
"name": args.name, "task": args.task,
|
||
"repo": args.repo or cfg.get("repo"),
|
||
"interval": args.interval, "last_run": 0})
|
||
save_state(args.state, state)
|
||
print(f"automation added: {args.name} every {args.interval}s")
|
||
else:
|
||
print(json.dumps(state["automations"], indent=2,
|
||
ensure_ascii=False))
|
||
elif args.cmd == "run-due":
|
||
now = time.time()
|
||
for auto in state["automations"]:
|
||
if now - auto.get("last_run", 0) < auto["interval"]:
|
||
log_event(args.state,
|
||
f"run-due skip {auto['name']} "
|
||
f"(interval {auto['interval']}s, not due)")
|
||
continue
|
||
name = f"{auto['name']}-{int(now)}"
|
||
task = dispatch(srv, state, name, auto["task"], auto["repo"],
|
||
args.model, args.effort, args.sandbox,
|
||
args.approval_policy)
|
||
save_state(args.state, state)
|
||
result = run_task_turn(srv, state, task, auto["task"],
|
||
args.model, args.effort, args.sandbox,
|
||
timeout=args.turn_timeout,
|
||
interrupt_file=_interrupt_request_path(args.state, name))
|
||
auto["last_run"] = now
|
||
if result["status"] != "completed":
|
||
notify_failure(args.state, cfg, name, result)
|
||
log_event(args.state,
|
||
f"run-due {auto['name']} -> {name}: "
|
||
f"{result['status']}")
|
||
print(f"automation {auto['name']} -> {name}: "
|
||
f"{result['status']} (进评审队列，见 host.py status/review)")
|
||
save_state(args.state, state)
|
||
finally:
|
||
srv.close()
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|