HA 阶段 1：worktree_host + 跨主机 /pr 队列化 (#50)

* 部署自动化：唯一发布链路 deploy.py + host_deploy.sh，sha/commit 双重验真，失败回滚

旧链路三次栽过的坑：通用 tar 路径导致旧包误部署、传输不验 sha、
主机脚本多份漂移、health 通过不等于新代码上线。本单把四点变硬门禁：
- 产物唯一命名 codex-app-server-<commit>.tar.gz + .sha256，远端唯一路径
- git archive 产物 + DEPLOYED_COMMIT 清单，默认拒绝脏工作区
- 传输链 scp -> scp_run -> chunk 分片，每路传完远端验 sha，不符删掉重来
- 主机侧唯一脚本：先验 sha/清单，staging 解包 + 编译检查，换目录上线，
  work/ 原样保留，health + 上线 commit 不符即回滚并退出非零
- deploy.py 成功后独立 ssh 再读一次 live 清单二次核对
- CI：host-verify 加 deploy/tests；release.yml 只在手动/tag + production
  environment 审批时部署，主干推送绝不自动部署生产
- deploy.sh 标记为旧链路弃用
- health 的 commit 字段已在父提交 HA 阶段 0 里随 server.py 落地，
  本分支整体包含它；commit 为空时只靠文件核对，门禁同样成立
测试：host 32、cloud 487（4 跳过）、deploy 9，全部通过；
host_deploy.sh 仿真覆盖成功/坏 sha/清单不符/health commit 不符回滚。
生产实机部署未做，等确认后再用新链路推。

* HA stage 1: worktree_host + queued cross-host PR jobs

---------

Co-authored-by: host <host@test.local>

cloud/autoscaler.py

@@ -34,6 +34,7 @@ import json
 import math
 import os
 import signal
+import socket
 import subprocess
 import sys
 import time
@@ -160,6 +161,13 @@ class Autoscaler:
             cmd.extend(["--db", self.db_path])
         cmd.extend(["--worker-id", worker_id,
                     "--worker-capacity", str(self.per_worker_capacity)])
+        # HA 阶段 1：宿主名强制追加（末尾胜出）。scaler 跑在宿主上，
+        # socket.gethostname() 就是宿主名；容器内 worker 若用自己的
+        # hostname 会拿到容器名，worktree_host 就错了，所以这里必须
+        # 显式传，不能让 worker 自己猜。
+        if not any(a == "--host-name" or a.startswith("--host-name=")
+                   for a in self.worker_args):
+            cmd.extend(["--host-name", socket.gethostname()])
         return cmd
 
     def _start_worker(self, worker_id):

cloud/db.py

@@ -911,7 +911,8 @@ def init_postgres_schema(conn):
     tcols = {row[1] for row in conn.execute(
         "PRAGMA table_info(tasks)").fetchall()}
     for _name, _type in (("executor", "TEXT"), ("pr_url", "TEXT"),
-                         ("pr_number", "INTEGER")):
+                         ("pr_number", "INTEGER"),
+                         ("worktree_host", "TEXT")):
         if _name not in tcols:
             conn.execute(
                 "ALTER TABLE tasks ADD COLUMN %s %s" % (_name, _type))

cloud/pr_exec.py

@@ -0,0 +1,233 @@
+"""跨主机 PR 执行（HA 阶段 1）。
+
+背景：`/pr` 原来只在 server 进程内同步跑，worktree 从 server 本机
+文件系统解析——server 与 worker 同主机是隐含前提。多机下任务的
+worktree 可能在另一台主机上，server 本机根本没有那个目录。
+
+本模块提供两条路（与 server 的 `/pr` 端点、worker 的 pr 任务循环
+共用，逻辑只写一次）：
+
+- 同步路（worktree 在本机）：`execute_task_pr` 直接跑，与旧端点
+  行为一致（写回 pr_url/pr_number、记审计 pr_created、成功发
+  review-requested、落盘失败发 merge-failed、跳过不发）。
+- 队列路（worktree 在别机）：`enqueue_pr_job` 把任务推进 `pr_jobs`
+  表；持有 worktree 的主机上的 local worker 每轮用 CAS 认领并跑
+  同一个 `execute_task_pr`，结果同样写回任务行。server 与 worker
+  之间不走新网络通道（不引 SSH 密钥分发），只复用已有的 DB 队列。
+
+只有 local worker 跑 pr 任务：host-worktrees/ 与 tenants/ 在主机上
+都可见（server 今天就是这么给 Docker 任务做 /pr 的），而容器内
+worker 看不到 host-worktrees/，也不一定有 gh。Docker worker 跳过
+pr 任务循环。
+"""
+
+import logging
+import time
+import uuid
+
+from cloud.auth import AuthStore
+from cloud.db import connect as _db_connect
+from cloud.github_app import resolve_app_token
+from cloud.pr import (PrSkipped, PrStepError, create_task_pr,
+                      parse_git_remote_owner_repo)
+from cloud.push import dispatch_event as _push_dispatch
+
+_log = logging.getLogger("cloud.pr_exec")
+
+
+def ensure_pr_jobs_table(conn):
+    """幂等建 pr_jobs 表（workers/webhook_deliveries 同款懒建表模式）。
+
+    PG 适配层把 `?` 译成 `%s`、把 AUTOINCREMENT 译成 IDENTITY，
+    因此同一条 DDL 在 SQLite 与 Postgres 下都能跑；调用方在
+    enqueue/claim 前各调一次即可。
+    """
+    conn.execute(
+        "CREATE TABLE IF NOT EXISTS pr_jobs("
+        "id TEXT PRIMARY KEY, task_id TEXT NOT NULL, "
+        "project_id TEXT NOT NULL, host TEXT NOT NULL, "
+        "status TEXT NOT NULL, pr_url TEXT, pr_number INTEGER, "
+        "error TEXT, created_at REAL, updated_at REAL, "
+        "started_at REAL, finished_at REAL)"
+    )
+    conn.commit()
+
+
+def find_open_pr_job(conn, task_id):
+    """找该任务未完成的 pr 任务（去重：同一任务同时只排一个）。"""
+    ensure_pr_jobs_table(conn)
+    row = conn.execute(
+        "SELECT * FROM pr_jobs WHERE task_id = ?"
+        " AND status IN ('queued', 'running')"
+        " ORDER BY created_at ASC LIMIT 1",
+        (task_id,)).fetchone()
+    if row is None:
+        return None
+    try:
+        return dict(row)
+    except TypeError:
+        return {"id": row[0], "task_id": row[1], "project_id": row[2],
+                "host": row[3], "status": row[4]}
+
+
+def enqueue_pr_job(conn, task_id, project_id, host):
+    """排一个 pr 任务（调用方已做去重）。返回 job id。"""
+    ensure_pr_jobs_table(conn)
+    job_id = uuid.uuid4().hex
+    now = time.time()
+    conn.execute(
+        "INSERT INTO pr_jobs(id, task_id, project_id, host, status,"
+        " created_at, updated_at) VALUES(?, ?, ?, ?, 'queued', ?, ?)",
+        (job_id, task_id, project_id, host, now, now))
+    conn.commit()
+    return job_id
+
+
+def claim_pr_job(conn, host):
+    """CAS 认领本主机的一个 queued pr 任务（最老优先）。
+
+    两个同主机 worker 抢同一任务时只有一个能拿到 rowcount == 1。
+    返回任务 dict 或 None。
+    """
+    ensure_pr_jobs_table(conn)
+    row = conn.execute(
+        "SELECT * FROM pr_jobs WHERE host = ? AND status = 'queued'"
+        " ORDER BY created_at ASC LIMIT 1",
+        (host,)).fetchone()
+    if row is None:
+        return None
+    job_id = row["id"] if hasattr(row, "__getitem__") else row[0]
+    now = time.time()
+    cur = conn.execute(
+        "UPDATE pr_jobs SET status = 'running', started_at = ?,"
+        " updated_at = ? WHERE id = ? AND status = 'queued'",
+        (now, now, job_id))
+    conn.commit()
+    if cur.rowcount != 1:
+        return None
+    try:
+        return dict(row)
+    except TypeError:
+        return {"id": row[0], "task_id": row[1], "project_id": row[2],
+                "host": row[3], "status": "running"}
+
+
+def finish_pr_job(conn, job_id, status, pr_url=None, pr_number=None,
+                  error=None):
+    """写回 pr 任务终态（completed/failed）。"""
+    ensure_pr_jobs_table(conn)
+    now = time.time()
+    conn.execute(
+        "UPDATE pr_jobs SET status = ?, pr_url = ?, pr_number = ?,"
+        " error = ?, finished_at = ?, updated_at = ? WHERE id = ?",
+        (status, pr_url, pr_number, error, now, now, job_id))
+    conn.commit()
+
+
+def _notify(project_id, event_type, payload, db_path):
+    """PR 事件推送（best-effort，与 server._notify_pr_event 同语义）。"""
+    try:
+        result = _push_dispatch(db_path, event_type, project_id, payload)
+    except Exception as exc:  # noqa: BLE001 -- 通知失败绝不炸 PR
+        _log.warning("pr notify %s failed: %s", event_type, exc)
+        return
+    if result.get("errors"):
+        _log.warning("pr notify %s errors: %s", event_type, result["errors"])
+
+
+def execute_task_pr(db_path, task_id, user_id, worktrees_root=None,
+                    git_runner=None, gh_runner=None, gh_which=None,
+                    app_token_client=None, github_api=None):
+    """给已 merged 的任务建 PR（同步执行体，server 端点与 worker 共用）。
+
+    步骤与旧 `_task_pr` 端点内联的逻辑一致：查项目 auto_pr/repo_url、
+    有 App 安装记录时现换 installation token 走 App 路径、否则走
+    gh + PAT；成功写回任务 pr_url/pr_number、记审计 pr_created、
+    发 review-requested；PrStepError 时发 merge-failed 后原样抛出；
+    PrSkipped 时不发任何事件、原样抛出。调用方按旧端点语义映射
+    响应（PrSkipped -> 200 pr_skipped，PrStepError -> 502）。
+
+    user_id：审计 pr_created 的操作人；server 端点传调用方，
+    worker 的 pr 任务循环传 "system"（与 sweep 同款约定）。
+    token 只在本次调用内存里出现，不进库、不进日志。
+    """
+    conn = _db_connect(str(db_path))
+    try:
+        row = conn.execute(
+            "SELECT * FROM tasks WHERE id = ?", (task_id,)).fetchone()
+        if row is None:
+            raise PrSkipped("task_not_found")
+        task = dict(row)
+        prow = conn.execute(
+            "SELECT auto_pr, repo_url FROM projects WHERE id = ?",
+            (task["project_id"],)).fetchone()
+    finally:
+        conn.close()
+    if task["status"] != "merged":
+        raise PrSkipped("not_merged")
+    auto_pr = 0
+    repo_url = None
+    if prow is not None:
+        try:
+            auto_pr = prow["auto_pr"] or 0
+        except (KeyError, IndexError, TypeError):
+            auto_pr = 0
+        try:
+            repo_url = prow["repo_url"]
+        except (KeyError, IndexError, TypeError):
+            repo_url = None
+    app_token = None
+    auth_via = "pat"
+    repo_full_name = None
+    if repo_url:
+        parsed = parse_git_remote_owner_repo(repo_url)
+        if parsed:
+            repo_full_name = "%s/%s" % parsed
+    if repo_full_name:
+        conn = _db_connect(str(db_path))
+        try:
+            app_token = resolve_app_token(
+                conn, repo_full_name, http_client=app_token_client)
+        except Exception:
+            app_token = None
+        finally:
+            conn.close()
+        if app_token:
+            auth_via = "app"
+    try:
+        result = create_task_pr(
+            task_id, auto_pr,
+            worktrees_root=worktrees_root,
+            git_runner=git_runner, gh_runner=gh_runner, gh_which=gh_which,
+            app_token=app_token, github_api=github_api)
+    except PrSkipped:
+        raise
+    except PrStepError as exc:
+        _notify(task["project_id"], "merge-failed",
+                {"task_id": task_id, "step": exc.step,
+                 "error": exc.message}, db_path)
+        raise
+    now = time.time()
+    conn = _db_connect(str(db_path))
+    try:
+        conn.execute(
+            "UPDATE tasks SET pr_url = ?, pr_number = ?,"
+            " updated_at = ? WHERE id = ?",
+            (result["pr_url"], result["pr_number"], now, task_id))
+        conn.commit()
+    finally:
+        conn.close()
+    store = AuthStore(db_path)
+    try:
+        store.audit(user_id, task["project_id"], "pr_created",
+                    result["pr_url"])
+    finally:
+        try:
+            store.conn.close()
+        except Exception:
+            pass
+    _notify(task["project_id"], "review-requested",
+            {"task_id": task_id, "pr_url": result["pr_url"],
+             "pr_number": result["pr_number"]}, db_path)
+    return {"pr_url": result["pr_url"], "pr_number": result["pr_number"],
+            "branch": result["branch"], "auth_via": auth_via}

cloud/server.py

@@ -42,6 +42,7 @@ import html
 import json
 import logging
 import os
+import socket
 import sqlite3
 import threading
 import time
@@ -61,8 +62,9 @@ from cloud.db import PostgresIntegrityError as _PgIntegrityError
 from cloud.db import PostgresOperationalError as _PgOperationalError
 from cloud.kb import KBStore
 from cloud.memory import MemoryStore
-from cloud.pr import (PrSkipped, PrStepError, create_task_pr,
-                      parse_git_remote_owner_repo)
+from cloud.pr import PrSkipped, PrStepError
+from cloud.pr_exec import (enqueue_pr_job, execute_task_pr,
+                           find_open_pr_job)
 from cloud.workers import (
     heartbeat_worker as _registry_heartbeat,
     list_workers as _registry_list,
@@ -103,7 +105,6 @@ from cloud.github_app import (
     record_installation_repos,
     remove_installation,
     remove_installation_repos,
-    resolve_app_token,
     save_app_config,
     set_fix_review_task_id,
 )
@@ -322,6 +323,9 @@ def migrate_tasks_table(conn):
         "goal": "TEXT",
         "verify_command": "TEXT",
         "goal_max_rounds": "INTEGER",
+        # HA 阶段 1：认领任务的 worker 所在主机名；/pr 据此判断
+        # worktree 在本机（同步跑）还是别机（排 pr 任务给该主机）。
+        "worktree_host": "TEXT",
     }
     for name, coltype in new_cols.items():
         if name not in cols:
@@ -456,6 +460,7 @@ def _task_to_dict(row):
         "goal": _row_get(row, "goal"),
         "verify_command": _row_get(row, "verify_command"),
         "goal_max_rounds": _row_get(row, "goal_max_rounds"),
+        "worktree_host": _row_get(row, "worktree_host"),
     }
 
 
@@ -1638,6 +1643,10 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
         成功时把 pr_url/pr_number 写回任务并记审计 pr_created。
         推送：成功时发 review-requested，落盘失败时发 merge-failed
         （均为 best-effort，不影响 PR 响应；跳过时不发）。
+        HA 阶段 1：任务的 worktree_host 与本机不同时，不直接跑，
+        而是排 pr 任务给该主机（返回 200 + pr_queued，同一任务
+        未完成的任务去重），由该主机上的 local worker 执行；
+        worktree 在本机（或未记录）时同步跑，与旧行为一致。
         App-token：项目 repo_url 能解析出 github.com 仓库名且
         库里有该仓库的 App 安装记录时，现换 installation token
         走 App 路径（响应里 auth_via=app），否则走原来的 gh +
@@ -1660,51 +1669,36 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
                 "status": task["status"],
             })
             return
-        conn = self._db()
-        try:
-            prow = conn.execute(
-                "SELECT auto_pr, repo_url FROM projects WHERE id = ?",
-                (task["project_id"],)).fetchone()
-        finally:
-            conn.close()
-        auto_pr = 0
-        repo_url = None
-        if prow is not None:
-            try:
-                auto_pr = prow["auto_pr"] or 0
-            except (KeyError, IndexError, TypeError):
-                auto_pr = 0
-            try:
-                repo_url = prow["repo_url"]
-            except (KeyError, IndexError, TypeError):
-                repo_url = None
-        app_token = None
-        auth_via = "pat"
-        repo_full_name = None
-        if repo_url:
-            parsed = parse_git_remote_owner_repo(repo_url)
-            if parsed:
-                repo_full_name = "%s/%s" % parsed
-        if repo_full_name:
+        # HA 阶段 1：worktree 在别机时不直接跑，排 pr 任务给该主机；
+        # 本机（或 worktree_host 未记录的老任务）走同步路。
+        worktree_host = task.get("worktree_host")
+        if worktree_host and worktree_host != socket.gethostname():
             conn = self._db()
             try:
-                app_token = resolve_app_token(
-                    conn, repo_full_name,
-                    http_client=getattr(
-                        self.server, "pr_app_token_client", None))
+                existing = find_open_pr_job(conn, task_id)
+                if existing is not None:
+                    job_id = existing["id"]
+                else:
+                    job_id = enqueue_pr_job(
+                        conn, task_id, task["project_id"], worktree_host)
             finally:
                 conn.close()
-            if app_token:
-                auth_via = "app"
+            self._send_json(200, {
+                "task_id": task_id, "status": task["status"],
+                "pr_queued": True, "pr_job_id": job_id,
+                "worktree_host": worktree_host,
+            })
+            return
         try:
-            result = create_task_pr(
-                task_id, auto_pr,
+            result = execute_task_pr(
+                self.server.db_path, task_id, user_id,
                 worktrees_root=getattr(
                     self.server, "host_worktrees_dir", None),
                 git_runner=getattr(self.server, "pr_git_runner", None),
                 gh_runner=getattr(self.server, "pr_gh_runner", None),
                 gh_which=getattr(self.server, "pr_gh_which", None),
-                app_token=app_token,
+                app_token_client=getattr(
+                    self.server, "pr_app_token_client", None),
                 github_api=getattr(self.server, "pr_github_api", None))
         except PrSkipped as exc:
             self._send_json(200, {
@@ -1713,41 +1707,24 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
             })
             return
         except PrStepError as exc:
-            self._notify_pr_event(
-                task["project_id"], "merge-failed",
-                {"task_id": task_id, "step": exc.step,
-                 "error": exc.message})
+            # merge-failed 事件已在 execute_task_pr 内发出，这里只映射响应。
             self._send_json(502, {
                 "error": exc.message, "step": exc.step,
                 "task_id": task_id, "status": task["status"],
             })
             return
-        now = time.time()
         conn = self._db()
         try:
-            conn.execute(
-                "UPDATE tasks SET pr_url = ?, pr_number = ?,"
-                " updated_at = ? WHERE id = ?",
-                (result["pr_url"], result["pr_number"], now, task_id),
-            )
-            conn.commit()
             task = _task_to_dict(
                 conn.execute(
                     "SELECT * FROM tasks WHERE id = ?",
                     (task_id,)).fetchone())
         finally:
             conn.close()
-        with _auth_store(self.server.db_path) as store:
-            store.audit(user_id, task["project_id"], "pr_created",
-                        result["pr_url"])
-        self._notify_pr_event(
-            task["project_id"], "review-requested",
-            {"task_id": task_id, "pr_url": result["pr_url"],
-             "pr_number": result["pr_number"]})
         self._send_json(200, dict(task, pr_url=result["pr_url"],
                                   pr_number=result["pr_number"],
                                   branch=result["branch"],
-                                  auth_via=auth_via))
+                                  auth_via=result["auth_via"]))
 
     def _heartbeat_task(self, user_id, task_id):
         row = self._fetch_task(task_id)

cloud/tests/test_pr_jobs.py

@@ -0,0 +1,317 @@
+"""HA 阶段 1 单测：pr 任务队列 + worktree_host + /pr 跨主机分支。
+
+覆盖：pr_jobs 的 enqueue/claim(CAS)/finish 与去重、tasks 的
+worktree_host 迁移（server 与 worker 两支）、认领时写主机名、
+/pr 在 worktree_host 为别机时排队（不直接跑、不重复排）、
+local worker 的 pr 任务循环（CAS 认领 -> 执行 -> 写回终态）、
+autoscaler 给 worker 强制追加 --host-name。
+
+git/GitHub 一律 fake 或 monkeypatch，不跑真实命令、不联网。
+"""
+
+import http.client
+import json
+import os
+import socket
+import sqlite3
+import sys
+import tempfile
+import threading
+import unittest
+from pathlib import Path
+from types import SimpleNamespace
+
+try:
+    from cloud.auth import AuthStore
+    from cloud.autoscaler import Autoscaler
+    from cloud.db import connect as _db_connect
+    from cloud.pr_exec import (claim_pr_job, enqueue_pr_job,
+                               find_open_pr_job, finish_pr_job)
+    from cloud.server import create_server, migrate_tasks_table
+    from cloud.worker import Worker
+except ImportError:
+    sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
+    from cloud.auth import AuthStore
+    from cloud.autoscaler import Autoscaler
+    from cloud.db import connect as _db_connect
+    from cloud.pr_exec import (claim_pr_job, enqueue_pr_job,
+                               ensure_pr_jobs_table, find_open_pr_job,
+                               finish_pr_job)
+    from cloud.server import create_server, migrate_tasks_table
+    from cloud.worker import Worker
+
+
+def _old_tasks_table(db_path):
+    conn = sqlite3.connect(db_path)
+    try:
+        conn.execute(
+            "CREATE TABLE tasks(id TEXT PRIMARY KEY, project_id TEXT,"
+            " prompt TEXT, status TEXT, depends_on TEXT, priority INTEGER,"
+            " created_by TEXT, created_at REAL, updated_at REAL,"
+            " last_heartbeat REAL)")
+        conn.commit()
+    finally:
+        conn.close()
+
+
+def _merged_task(db_path, task_id, project_id, host=None):
+    conn = sqlite3.connect(db_path)
+    try:
+        conn.execute(
+            "INSERT INTO tasks(id, project_id, prompt, status,"
+            " depends_on, priority, created_by, created_at,"
+            " updated_at, last_heartbeat) VALUES(?, ?, ?, 'merged',"
+            " '[]', 0, 'system', 0, 0, NULL)",
+            (task_id, project_id, "do it"))
+        if host is not None:
+            conn.execute("UPDATE tasks SET worktree_host = ? WHERE id = ?",
+                         (host, task_id))
+        conn.commit()
+    finally:
+        conn.close()
+
+
+class PrJobsTableTestCase(unittest.TestCase):
+    def setUp(self):
+        self.tmp = tempfile.TemporaryDirectory()
+        self.db_path = os.path.join(self.tmp.name, "control.db")
+
+    def tearDown(self):
+        self.tmp.cleanup()
+
+    def test_enqueue_claim_finish_and_cas(self):
+        conn = _db_connect(self.db_path)
+        try:
+            job1 = enqueue_pr_job(conn, "t1", "p1", "host-a")
+            job2 = enqueue_pr_job(conn, "t2", "p1", "host-a")
+            self.assertIsNotNone(find_open_pr_job(conn, "t1"))
+            claimed = claim_pr_job(conn, "host-a")
+            self.assertEqual(claimed["task_id"], "t1")
+            # 第二次认领拿不到同一任务（CAS），只能拿到下一个。
+            claimed2 = claim_pr_job(conn, "host-a")
+            self.assertEqual(claimed2["task_id"], "t2")
+            self.assertIsNone(claim_pr_job(conn, "host-a"))
+            # 别的主机认领不到 host-a 的任务。
+            self.assertIsNone(claim_pr_job(conn, "host-b"))
+            finish_pr_job(conn, job1, "completed",
+                          "https://github.com/o/a/pull/1", 1)
+            self.assertIsNone(find_open_pr_job(conn, "t1"))
+            row = conn.execute(
+                "SELECT status, pr_url, pr_number FROM pr_jobs"
+                " WHERE id = ?", (job1,)).fetchone()
+            self.assertEqual(row[0], "completed")
+            self.assertEqual(row[1], "https://github.com/o/a/pull/1")
+            self.assertEqual(row[2], 1)
+        finally:
+            conn.close()
+
+
+class WorktreeHostMigrationTestCase(unittest.TestCase):
+    def setUp(self):
+        self.tmp = tempfile.TemporaryDirectory()
+        self.db_path = os.path.join(self.tmp.name, "control.db")
+        _old_tasks_table(self.db_path)
+
+    def tearDown(self):
+        self.tmp.cleanup()
+
+    def test_server_migrate_adds_worktree_host(self):
+        conn = _db_connect(self.db_path)
+        try:
+            migrate_tasks_table(conn)
+            cols = {row[1] for row in conn.execute(
+                "PRAGMA table_info(tasks)").fetchall()}
+            self.assertIn("worktree_host", cols)
+            # 再跑一次是 no-op。
+            migrate_tasks_table(conn)
+        finally:
+            conn.close()
+
+    def test_worker_ensure_adds_worktree_host(self):
+        worker = Worker(self.db_path, "/nonexistent/host.py",
+                        env_key="TEST_KEY")
+        conn = _db_connect(self.db_path)
+        try:
+            worker._ensure_result_columns(conn)
+            cols = {row[1] for row in conn.execute(
+                "PRAGMA table_info(tasks)").fetchall()}
+            self.assertIn("worktree_host", cols)
+        finally:
+            conn.close()
+
+    def test_claim_writes_host_name(self):
+        worker = Worker(self.db_path, "/nonexistent/host.py",
+                        env_key="TEST_KEY", host_name="worker-host-9")
+        conn = _db_connect(self.db_path)
+        try:
+            conn.execute(
+                "INSERT INTO tasks(id, project_id, prompt, status,"
+                " depends_on, priority, created_by, created_at,"
+                " updated_at, last_heartbeat) VALUES('t1', 'p1', 'x',"
+                " 'queued', '[]', 0, 'system', 0, 0, NULL)")
+            conn.commit()
+        finally:
+            conn.close()
+        self.assertTrue(worker._claim("t1", executor="local"))
+        conn = _db_connect(self.db_path)
+        try:
+            row = conn.execute(
+                "SELECT status, worktree_host FROM tasks WHERE id = 't1'"
+            ).fetchone()
+            self.assertEqual(row[0], "running")
+            self.assertEqual(row[1], "worker-host-9")
+        finally:
+            conn.close()
+
+
+class PrEndpointRemoteTestCase(unittest.TestCase):
+    def setUp(self):
+        self.tmp = tempfile.TemporaryDirectory()
+        self.db_path = os.path.join(self.tmp.name, "control.db")
+        self.auth = AuthStore(self.db_path)
+        self.owner_id, self.owner_key = self.auth.create_user("owner")
+        self.server = create_server(self.db_path, "127.0.0.1", 0)
+        self.port = self.server.server_address[1]
+        self.thread = threading.Thread(
+            target=self.server.serve_forever, daemon=True)
+        self.thread.start()
+
+    def tearDown(self):
+        self.server.shutdown()
+        self.server.server_close()
+        self.thread.join(timeout=5)
+        try:
+            self.auth.conn.close()
+        except Exception:
+            pass
+        self.tmp.cleanup()
+
+    def _req(self, method, path, body=None, key=None):
+        conn = http.client.HTTPConnection(
+            "127.0.0.1", self.port, timeout=10)
+        hdrs = {}
+        payload = None
+        if body is not None:
+            payload = json.dumps(body).encode("utf-8")
+            hdrs["Content-Type"] = "application/json"
+        if key is not None:
+            hdrs["X-API-Key"] = key
+        try:
+            conn.request(method, path, body=payload, headers=hdrs)
+            resp = conn.getresponse()
+            data = resp.read()
+        finally:
+            conn.close()
+        try:
+            parsed = json.loads(data.decode("utf-8")) if data else None
+        except ValueError:
+            parsed = None
+        return resp.status, parsed
+
+    def test_remote_host_queues_and_dedupes(self):
+        status, project = self._req(
+            "POST", "/projects", {"name": "p", "auto_pr": 1},
+            self.owner_key)
+        self.assertEqual(status, 201, project)
+        # 项目成员默认 owner 即 maintainer（create 时建成员行）。
+        _merged_task(self.db_path, "t-remote", project["id"],
+                     host="elsewhere-host-1")
+        status, body = self._req(
+            "POST", "/tasks/t-remote/pr", {}, self.owner_key)
+        self.assertEqual(status, 200, body)
+        self.assertTrue(body.get("pr_queued"))
+        self.assertEqual(body.get("worktree_host"), "elsewhere-host-1")
+        job_id = body.get("pr_job_id")
+        self.assertIsNotNone(job_id)
+        # 再调一次：去重，返回同一个 job，不重复排队。
+        status, body2 = self._req(
+            "POST", "/tasks/t-remote/pr", {}, self.owner_key)
+        self.assertEqual(status, 200, body2)
+        self.assertEqual(body2.get("pr_job_id"), job_id)
+        conn = _db_connect(self.db_path)
+        try:
+            rows = conn.execute(
+                "SELECT COUNT(*) FROM pr_jobs WHERE task_id = 't-remote'"
+            ).fetchone()
+            self.assertEqual(rows[0], 1)
+        finally:
+            conn.close()
+
+
+class WorkerPrJobsTestCase(unittest.TestCase):
+    def setUp(self):
+        self.tmp = tempfile.TemporaryDirectory()
+        self.db_path = os.path.join(self.tmp.name, "control.db")
+        _old_tasks_table(self.db_path)
+
+    def tearDown(self):
+        self.tmp.cleanup()
+
+    def test_local_worker_runs_pr_job(self):
+        import cloud.worker as worker_mod
+        calls = []
+
+        def _fake_execute(db_path, task_id, user_id, **kwargs):
+            calls.append((task_id, user_id))
+            return {"pr_url": "https://github.com/o/a/pull/9",
+                    "pr_number": 9, "branch": "host/t1",
+                    "auth_via": "pat"}
+
+        worker = Worker(self.db_path, "/nonexistent/host.py",
+                        env_key="TEST_KEY", host_name="elsewhere-host-1")
+        conn = _db_connect(self.db_path)
+        try:
+            worker._ensure_result_columns(conn)
+            conn.execute(
+                "INSERT INTO tasks(id, project_id, prompt, status,"
+                " depends_on, priority, created_by, created_at,"
+                " updated_at, last_heartbeat) VALUES('t1', 'p1', 'x',"
+                " 'merged', '[]', 0, 'system', 0, 0, NULL)")
+            conn.commit()
+            enqueue_pr_job(conn, "t1", "p1", "elsewhere-host-1")
+            enqueue_pr_job(conn, "t2", "p1", "other-host")
+        finally:
+            conn.close()
+        orig = worker_mod.execute_task_pr
+        worker_mod.execute_task_pr = _fake_execute
+        try:
+            results = worker._run_pr_jobs()
+        finally:
+            worker_mod.execute_task_pr = orig
+        self.assertEqual(len(results), 1)
+        self.assertEqual(results[0]["task_id"], "t1")
+        self.assertEqual(results[0]["status"], "completed")
+        self.assertEqual(calls, [("t1", "system")])
+        conn = _db_connect(self.db_path)
+        try:
+            row = conn.execute(
+                "SELECT status, pr_url FROM pr_jobs WHERE task_id = 't1'"
+            ).fetchone()
+            self.assertEqual(row[0], "completed")
+            self.assertEqual(row[1], "https://github.com/o/a/pull/9")
+            # 别的主机的任务没被碰。
+            row2 = conn.execute(
+                "SELECT status FROM pr_jobs WHERE task_id = 't2'"
+            ).fetchone()
+            self.assertEqual(row2[0], "queued")
+        finally:
+            conn.close()
+
+    def test_docker_worker_skips_pr_jobs(self):
+        from cloud.sandbox import DockerExecutor
+        worker = Worker(self.db_path, "/nonexistent/host.py",
+                        env_key="TEST_KEY",
+                        executor=DockerExecutor("/fake/host.py"))
+        conn = _db_connect(self.db_path)
+        try:
+            enqueue_pr_job(conn, "t1", "p1", worker.host_name)
+        finally:
+            conn.close()
+        self.assertEqual(worker._run_pr_jobs(), [])
+
+    def test_autoscaler_forces_host_name(self):
+        scaler = Autoscaler(self.db_path, [])
+        cmd = scaler._worker_command("w1")
+        self.assertIn("--host-name", cmd)
+        self.assertEqual(cmd[cmd.index("--host-name") + 1],
+                         socket.gethostname())

cloud/worker.py

@@ -129,6 +129,8 @@ from cloud.cost import CostStore, estimate_tokens, price_usd_micros, sum_usage
 from cloud.db import connect as _db_connect
 from cloud.logging_config import get_logger, log_event, setup_logging
 from cloud.models import ModelRouter, is_fallbackable, record_fallback
+from cloud.pr import PrSkipped, PrStepError
+from cloud.pr_exec import (claim_pr_job, execute_task_pr, finish_pr_job)
 from cloud.queue import budget_exceeded, ready_tasks
 from cloud.repos import RepoError, resolve_repo
 from cloud.sandbox import CloudExecutor, DockerExecutor, LocalExecutor
@@ -245,7 +247,8 @@ class Worker:
                  worker_id=None, capacity=1, fallback_models=None,
                  tenant_isolation=False, tenants_root=None,
                  tenant_template_home=None, tenant_template_repo=None,
-                 egress_proxy_port=0, claim_executor_only=False):
+                 egress_proxy_port=0, claim_executor_only=False,
+                 host_name=None, pr_jobs_only=False):
         self.db_path = str(db_path)
         self.host_py_path = host_py_path
         self.config_path = config_path
@@ -338,6 +341,15 @@ class Worker:
         # 容器的 NULL 任务（那些任务在宿主上跑就丢了 Docker
         # 外层，违背“按任务二选一”的路由意图）。
         self.claim_executor_only = bool(claim_executor_only)
+        # HA 阶段 1：本 worker 所在主机名。认领任务时写进
+        # tasks.worktree_host，/pr 据此判断 worktree 在哪台主机。
+        # 容器内 worker 的 socket.gethostname() 是容器名、不是宿主名，
+        # 所以 autoscaler 必须显式传 --host-name（宿主名），见
+        # autoscaler._worker_command；单机/测试默认本机名即可。
+        self.host_name = host_name or socket.gethostname()
+        # 只跑 pr 任务（HA 阶段 1 的跨主机 /pr 演练与专用 pr-runner
+        # 用）：不认领普通任务，只处理本主机的 pr_jobs。
+        self.pr_jobs_only = bool(pr_jobs_only)
         # 运维心跳：路径优先显式参数，其次环境变量，默认当前目录下
         # 的 worker-heartbeat.json；主机路径不在代码里硬编码，主机
         # 部署时用 --heartbeat-file 或环境变量传入（见 deploy/README）。
@@ -467,25 +479,28 @@ class Worker:
             if executor is None:
                 cur = conn.execute(
                     "UPDATE tasks SET status = 'running',"
-                    " last_heartbeat = ?, updated_at = ?"
+                    " last_heartbeat = ?, updated_at = ?,"
+                    " worktree_host = ?"
                     " WHERE id = ? AND status = 'queued'",
-                    (now, now, task_id),
+                    (now, now, self.host_name, task_id),
                 )
             elif self.claim_executor_only:
                 cur = conn.execute(
                     "UPDATE tasks SET status = 'running',"
-                    " last_heartbeat = ?, updated_at = ?"
+                    " last_heartbeat = ?, updated_at = ?,"
+                    " worktree_host = ?"
                     " WHERE id = ? AND status = 'queued'"
                     " AND executor = ?",
-                    (now, now, task_id, executor),
+                    (now, now, self.host_name, task_id, executor),
                 )
             else:
                 cur = conn.execute(
                     "UPDATE tasks SET status = 'running',"
-                    " last_heartbeat = ?, updated_at = ?"
+                    " last_heartbeat = ?, updated_at = ?,"
+                    " worktree_host = ?"
                     " WHERE id = ? AND status = 'queued'"
                     " AND (executor IS NULL OR executor = ?)",
-                    (now, now, task_id, executor),
+                    (now, now, self.host_name, task_id, executor),
                 )
             conn.commit()
             return cur.rowcount == 1
@@ -523,7 +538,8 @@ class Worker:
                 "executor": "TEXT", "pr_url": "TEXT",
                 "pr_number": "INTEGER", "goal": "TEXT",
                 "verify_command": "TEXT",
-                "goal_max_rounds": "INTEGER"}.items():
+                "goal_max_rounds": "INTEGER",
+                "worktree_host": "TEXT"}.items():
             if name not in cols:
                 try:
                     conn.execute(
@@ -1651,6 +1667,73 @@ class Worker:
                 "actual_model": actual_model,
                 "cost_record_id": cost_id}
 
+    def _run_pr_jobs(self):
+        """跑本主机的 pr 任务（HA 阶段 1），返回处理结果列表。
+
+        只有 local worker 跑：host-worktrees/ 与 tenants/ 在主机上
+        都可见，容器内 worker 看不到 host-worktrees/。每轮把本主机
+        的 queued 任务按 CAS 认领跑完（pr 任务稀少，串行即可）；
+        execute_task_pr 内部已写回任务 pr_url/pr_number、记审计、
+        发 review-requested/merge-failed，这里只写回任务终态。
+        跳过（PrSkipped，如任务已不是 merged）与落盘失败都记
+        failed + error，任务本身保持 merged、可重调 /pr 再排队。
+        """
+        if self._executor_name() != "local":
+            return []
+        results = []
+        while True:
+            conn = _db_connect(self.db_path)
+            try:
+                job = claim_pr_job(conn, self.host_name)
+            finally:
+                conn.close()
+            if job is None:
+                break
+            job_id = job["id"]
+            task_id = job["task_id"]
+            try:
+                result = execute_task_pr(
+                    self.db_path, task_id, "system")
+                conn = _db_connect(self.db_path)
+                try:
+                    finish_pr_job(conn, job_id, "completed",
+                                  result["pr_url"], result["pr_number"])
+                finally:
+                    conn.close()
+                results.append({"pr_job_id": job_id, "task_id": task_id,
+                                "status": "completed",
+                                "pr_url": result["pr_url"]})
+                log_event(self.logger, "pr_job_completed",
+                          task_id=task_id, status="completed")
+            except PrSkipped as exc:
+                err = "skipped: %s" % exc.reason
+                conn = _db_connect(self.db_path)
+                try:
+                    finish_pr_job(conn, job_id, "failed", error=err)
+                finally:
+                    conn.close()
+                results.append({"pr_job_id": job_id, "task_id": task_id,
+                                "status": "failed", "error": err})
+            except PrStepError as exc:
+                err = "%s: %s" % (exc.step, exc.message)
+                conn = _db_connect(self.db_path)
+                try:
+                    finish_pr_job(conn, job_id, "failed", error=err)
+                finally:
+                    conn.close()
+                results.append({"pr_job_id": job_id, "task_id": task_id,
+                                "status": "failed", "error": err})
+            except Exception as exc:  # noqa: BLE001 -- 任务失败不能炸循环
+                err = str(exc)
+                conn = _db_connect(self.db_path)
+                try:
+                    finish_pr_job(conn, job_id, "failed", error=err)
+                finally:
+                    conn.close()
+                results.append({"pr_job_id": job_id, "task_id": task_id,
+                                "status": "failed", "error": err})
+        return results
+
     def run_all(self, max_tasks=None):
         """循环认领执行，直到无就绪任务、预算超限或达到 max_tasks。
 
@@ -1668,6 +1751,11 @@ class Worker:
         停机信号（_loop_stop/stop_event）只阻止新认领，不会
         丢弃已提交的任务；排空等待由 run_loop 的 finally 负责。
         """
+        # pr 任务先跑（HA 阶段 1）：每轮先把本主机的 queued pr 任务
+        # 跑完再认领普通任务；pr_jobs_only 时只跑 pr 任务。
+        self._run_pr_jobs()
+        if self.pr_jobs_only:
+            return []
         if self.capacity <= 1:
             results = []
             while max_tasks is None or len(results) < max_tasks:
@@ -1960,6 +2048,14 @@ def main(argv=None):
                         help="worker 并发上限：同一 worker 内最多同时跑"
                              "几个任务（默认 1，即旧串行行为；>1 时用"
                              "线程池并行，认领仍是 CAS，成本写库持锁）")
+    parser.add_argument("--host-name", default=None,
+                        help="本 worker 所在主机名（HA 阶段 1）：认领任务"
+                             "时写进 tasks.worktree_host，/pr 据此判断"
+                             "worktree 在哪台主机；容器内 worker 必须由"
+                             "autoscaler 显式传宿主名，不能用容器名")
+    parser.add_argument("--pr-jobs-only", action="store_true",
+                        help="只跑 pr 任务（HA 阶段 1 的跨主机 /pr 演练"
+                             "与专用 pr-runner 用）：不认领普通任务")
     parser.add_argument("--docker-image", default="codex-app-server:latest")
     parser.add_argument("--docker-cpus", default="2")
     parser.add_argument("--docker-memory", default="2g")
@@ -2040,6 +2136,8 @@ def main(argv=None):
                     heartbeat_file=args.heartbeat_file,
                     worker_id=args.worker_id,
                     capacity=args.worker_capacity,
+                    host_name=args.host_name,
+                    pr_jobs_only=args.pr_jobs_only,
                     tenant_isolation=args.tenant_isolation,
                     tenants_root=args.tenants_root,
                     tenant_template_home=args.tenant_template_home,