host: phase2 daemon + health + auto-recovery (restore, recover, double-fork)

host.py

@@ -381,6 +381,172 @@ def _read_state_file(path):
     return {"tasks": {}, "automations": []}
 
 
+def load_state_strict(path):
+    """严格读 state：文件不存在返回 (空 state, None)；
+    文件存在但 JSON 坏了返回 (None, 错误信息)，绝不静默
+    当成空 state——否则损坏一次就等于全部任务丢了，
+    这是自动恢复要先堵住的洞（旧 _read_state_file 会吞掉
+    解析错误，health/recover 必须用这个严格版）。"""
+    if not os.path.exists(path):
+        return {"tasks": {}, "automations": []}, None
+    try:
+        with open(path) as f:
+            return json.load(f), None
+    except Exception as e:
+        return None, str(e)
+
+
+def restore_state(state_path):
+    """state 损坏或丢失时从备份恢复：按 .bak、.bak.1、.bak.2
+    顺序找第一个能解析的备份，先把坏文件（如果存在）改名
+    成 .corrupt.<时间戳> 留底，再把备份拷回主文件。
+    返回 (ok, message)。主文件完好时不做任何事。"""
+    strict, err = load_state_strict(state_path)
+    if err is None and strict is not None:
+        return True, "state 完好，无需恢复"
+    if os.path.exists(state_path):
+        corrupt = f"{state_path}.corrupt.{int(time.time())}"
+        try:
+            os.replace(state_path, corrupt)
+        except Exception as e:
+            return False, f"坏 state 改名留底失败：{e}"
+    else:
+        corrupt = None
+    for suffix in (".bak", ".bak.1", ".bak.2"):
+        cand = state_path + suffix
+        if not os.path.exists(cand):
+            continue
+        try:
+            with open(cand) as f:
+                data = json.load(f)
+        except Exception:
+            continue
+        import shutil
+        shutil.copy2(cand, state_path)
+        return True, (f"已从 {suffix} 恢复，version="
+                      f"{data.get('version')}，"
+                      f"坏文件留底：{corrupt}")
+    return False, "没有可用的备份（.bak/.bak.1/.bak.2 均缺失或损坏）"
+
+
+def recover_interrupted(state, state_path, older_than=3600, cfg=None):
+    """自动恢复（只标记，不自动重跑模型回合）：
+    status=running 且最后回合时间超过 older_than 秒的任务，
+    判定为进程被杀/VM 回收导致的中断，标 interrupted 并记日志，
+    用户随后可用 run 续跑（thread 在 CODEX_HOME 里不丢）。
+    不自动重跑是刻意的安全边界：自动重跑会不经用户同意
+    花中转站额度，daemon 只负责标记 + 日志 + 通知。
+    返回被标记的任务名列表。"""
+    now = time.time()
+    recovered = []
+    for name, t in (state.get("tasks") or {}).items():
+        if t.get("status") != "running":
+            continue
+        last = t.get("last_turn_at") or t.get("created_at") or 0
+        if now - last < older_than:
+            continue
+        t["status"] = "interrupted"
+        recovered.append(name)
+        try:
+            log_event(state_path, f"recover {name} -> interrupted "
+                                  f"(last_turn {int(now - last)}s ago)")
+        except Exception:
+            pass
+        if cfg is not None:
+            notify_failure(state_path, cfg, name,
+                           {"status": "interrupted",
+                            "text": "process killed / VM reclaimed"})
+    return recovered
+
+
+def run_health_check(state_path, cfg, codex_bin, codex_home, approval,
+                     server_factory=None):
+    """健康检查（不跑模型）：返回 report dict，ok 为全部通过。
+    检查项：二进制存在、配置可读、state 可解析、备份存在、
+    daemon pid 是否存活/过期、server 能 initialize + list、
+    缺失 worktree（doctor 的逻辑）。server_factory 供单测注入
+    假 server，避免健康检查本身要拉真进程。"""
+    checks = {}
+    checks["codex_bin_exists"] = bool(codex_bin) and os.path.exists(codex_bin)
+    checks["config_loads"] = isinstance(cfg, dict)
+    strict, err = load_state_strict(state_path)
+    checks["state_valid"] = err is None
+    checks["state_error"] = err
+    checks["backup_exists"] = os.path.exists(state_path + ".bak")
+    if strict is not None:
+        doc = check_state(state_path, strict)
+        checks["failed_tasks"] = doc["failed_tasks"]
+        checks["missing_worktrees"] = doc["missing_worktrees"]
+        checks["open_review_comments"] = doc["open_review_comments"]
+    else:
+        checks["failed_tasks"] = []
+        checks["missing_worktrees"] = []
+        checks["open_review_comments"] = 0
+    # daemon pid
+    pidfile = os.path.join(os.path.dirname(os.path.abspath(state_path)),
+                           "host-daemon.pid")
+    checks["daemon_pidfile"] = pidfile
+    if os.path.exists(pidfile):
+        try:
+            pid = int(open(pidfile).read().strip())
+            os.kill(pid, 0)
+            checks["daemon_alive"] = True
+            checks["daemon_pid"] = pid
+        except Exception:
+            checks["daemon_alive"] = False
+            checks["daemon_pid"] = None
+    else:
+        checks["daemon_alive"] = False
+        checks["daemon_pid"] = None
+    # server（无模型成本，只 initialize + list）
+    try:
+        if server_factory is not None:
+            srv = server_factory()
+        else:
+            srv = Server(codex_bin=codex_bin, codex_home=codex_home,
+                         approval=approval)
+        try:
+            srv.initialize()
+            threads = srv.list_threads()
+            checks["server_ok"] = True
+            checks["server_threads"] = len(
+                (threads or {}).get("data") or []) if isinstance(threads, dict) else None
+        finally:
+            try:
+                srv.close()
+            except Exception:
+                pass
+    except Exception as e:
+        checks["server_ok"] = False
+        checks["server_error"] = str(e)[:500]
+    ok = bool(checks["codex_bin_exists"] and checks["state_valid"]
+              and checks["server_ok"])
+    return {"ok": ok, "checks": checks}
+
+
+def daemonize(pidfile, logfile):
+    """双 fork 守护进程（WeChat 桥 daemon_serve.py 同款教训：
+    setsid nohup 会在 launcher 会话结束时被 runtime 按进程树
+    回收杀掉，双 fork 到 PID 1 才能活过会话结束）。
+    父进程返回 False（应退出），子进程返回 True（继续跑循环）。"""
+    if os.fork() > 0:
+        return False
+    os.setsid()
+    if os.fork() > 0:
+        os._exit(0)
+    sys.stdout.flush()
+    sys.stderr.flush()
+    log_f = open(logfile, "a", encoding="utf-8")
+    os.dup2(log_f.fileno(), sys.stdout.fileno())
+    os.dup2(log_f.fileno(), sys.stderr.fileno())
+    with open("/dev/null", "r") as devnull:
+        os.dup2(devnull.fileno(), sys.stdin.fileno())
+    with open(pidfile, "w") as f:
+        f.write(str(os.getpid()))
+    return True
+
+
+
 def load_state(path):
     with _state_lock(path, exclusive=False):
         st = _read_state_file(path)
@@ -843,10 +1009,14 @@ def run_task_turn(server, state, task, prompt, model, effort,
     # 否则用调用方显式传的 sandbox；两者都没有时不传 sandboxPolicy，
     # 保留 thread 原有设置（旧任务没有 sandbox 记录时尤其重要）。
     use_sandbox = task.get("sandbox") or sandbox
+    started = time.time()
     result = server.run_turn(task["thread_id"], prompt, model=model,
                              effort=effort, sandbox=use_sandbox)
+    finished = time.time()
     record_turn_usage(state, failed=(result["status"] != "completed"))
-    task["turns"].append({"prompt": prompt, "result": result})
+    task["turns"].append({"prompt": prompt, "result": result,
+                          "started_at": started, "finished_at": finished})
+    task["last_turn_at"] = finished
     task["status"] = "completed" if result["status"] == "completed" \
         else "failed"
     task["last_text"] = result.get("text", "")
@@ -939,6 +1109,18 @@ def main():
     p = sub.add_parser("fix-review"); p.add_argument("--task", required=True)
     sub.add_parser("doctor")
     sub.add_parser("failures")
+    sub.add_parser("health")
+    sub.add_parser("restore-state")
+    p = sub.add_parser("recover")
+    p.add_argument("--older-than", type=int, default=3600,
+                   help="最后回合超过多少秒未更新才判定中断，默认 3600")
+    p = sub.add_parser("daemon")
+    p.add_argument("--interval", type=int, default=300,
+                   help="每轮间隔秒数，默认 300")
+    p.add_argument("--foreground", action="store_true",
+                   help="前台跑循环（测试用，不双 fork）")
+    sub.add_parser("daemon-status")
+    sub.add_parser("daemon-stop")
     p = sub.add_parser("discard"); p.add_argument("--task", required=True)
     p = sub.add_parser("archive"); p.add_argument("--task", required=True)
     p = sub.add_parser("fork"); p.add_argument("--task", required=True); p.add_argument("--name", required=True); p.add_argument("--sandbox", default=None)
@@ -1053,6 +1235,131 @@ def main():
         save_state(args.state, state)
         print(json.dumps(c, indent=2, ensure_ascii=False))
         return
+    if args.cmd == "health":
+        report = run_health_check(args.state, cfg, args.codex_bin or default_codex_bin(),
+                                  args.codex_home, args.approval)
+        print(json.dumps(report, indent=2, ensure_ascii=False))
+        if not report["ok"]:
+            raise SystemExit(1)
+        return
+    if args.cmd == "restore-state":
+        ok, msg = restore_state(args.state)
+        print(msg)
+        if not ok:
+            raise SystemExit(1)
+        return
+    if args.cmd == "recover":
+        strict, err = load_state_strict(args.state)
+        if err is not None:
+            print(f"state 已损坏，recover 不会覆盖它，先跑 restore-state：{err}")
+            raise SystemExit(1)
+        recovered = recover_interrupted(state, args.state,
+                                        older_than=args.older_than, cfg=cfg)
+        if recovered:
+            save_state(args.state, state)
+        print(json.dumps({"recovered": recovered}, indent=2, ensure_ascii=False))
+        return
+    if args.cmd == "daemon-status":
+        pidfile = os.path.join(os.path.dirname(os.path.abspath(args.state)),
+                               "host-daemon.pid")
+        logfile = os.path.join(os.path.dirname(os.path.abspath(args.state)),
+                               "host-daemon.log")
+        alive = False
+        pid = None
+        if os.path.exists(pidfile):
+            try:
+                pid = int(open(pidfile).read().strip())
+                os.kill(pid, 0)
+                alive = True
+            except Exception:
+                alive = False
+        tail = ""
+        if os.path.exists(logfile):
+            try:
+                tail = "\n".join(open(logfile).read().splitlines()[-5:])
+            except Exception:
+                pass
+        print(json.dumps({"alive": alive, "pid": pid, "pidfile": pidfile,
+                          "log_tail": tail}, indent=2, ensure_ascii=False))
+        return
+    if args.cmd == "daemon-stop":
+        pidfile = os.path.join(os.path.dirname(os.path.abspath(args.state)),
+                               "host-daemon.pid")
+        if not os.path.exists(pidfile):
+            print("daemon 未运行（无 pidfile）")
+            return
+        try:
+            pid = int(open(pidfile).read().strip())
+        except Exception:
+            print("pidfile 损坏，直接删除")
+            os.remove(pidfile)
+            return
+        try:
+            os.kill(pid, 15)
+            for _ in range(10):
+                time.sleep(0.5)
+                try:
+                    os.kill(pid, 0)
+                except Exception:
+                    break
+            else:
+                os.kill(pid, 9)
+        except Exception:
+            pass
+        try:
+            os.remove(pidfile)
+        except Exception:
+            pass
+        print(f"daemon 已停：pid {pid}")
+        return
+    if args.cmd == "daemon":
+        state_dir = os.path.dirname(os.path.abspath(args.state))
+        pidfile = os.path.join(state_dir, "host-daemon.pid")
+        logfile = os.path.join(state_dir, "host-daemon.log")
+        if not args.foreground:
+            if os.path.exists(pidfile):
+                try:
+                    old_pid = int(open(pidfile).read().strip())
+                    os.kill(old_pid, 0)
+                    print(f"daemon 已在运行：pid {old_pid}，先 daemon-stop 再起")
+                    return
+                except Exception:
+                    pass
+            if not daemonize(pidfile, logfile):
+                print(f"daemon 已在后台启动：pidfile {pidfile}，log {logfile}")
+                return
+        # 子进程 / 前台：循环跑 recover + run-due，每轮都是独立
+        # 子进程调 host.py CLI，这样一轮崩了不带走 daemon 本体，
+        # 且 run-due 的模型回合数受自动化本身和预算控制。
+        import signal
+        stop = {"flag": False}
+
+        def _on_term(signum, frame):
+            stop["flag"] = True
+        signal.signal(signal.SIGTERM, _on_term)
+        signal.signal(signal.SIGINT, _on_term)
+        print(f"host daemon started pid={os.getpid()} interval={args.interval}s "
+              f"config={config_path} state={args.state}", flush=True)
+        while not stop["flag"]:
+            ts = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
+            print(f"{ts} daemon tick: recover + run-due", flush=True)
+            for subcmd in (["recover", "--older-than", "3600"], ["run-due"]):
+                try:
+                    subprocess.run([sys.executable, os.path.join(HERE, "host.py"),
+                                    "--config", config_path] + subcmd,
+                                   timeout=3600)
+                except Exception as e:
+                    print(f"{ts} daemon subcommand {subcmd} 失败：{e}", flush=True)
+            for _ in range(args.interval):
+                if stop["flag"]:
+                    break
+                time.sleep(1)
+        try:
+            os.remove(pidfile)
+        except Exception:
+            pass
+        print("host daemon stopped", flush=True)
+        return
     srv = Server(codex_bin=args.codex_bin, codex_home=args.codex_home,
                  approval=args.approval)
     try: