HA 阶段 1：worktree_host + 跨主机 /pr 队列化 (#50)
* 部署自动化：唯一发布链路 deploy.py + host_deploy.sh，sha/commit 双重验真，失败回滚 旧链路三次栽过的坑：通用 tar 路径导致旧包误部署、传输不验 sha、 主机脚本多份漂移、health 通过不等于新代码上线。本单把四点变硬门禁： - 产物唯一命名 codex-app-server-<commit>.tar.gz + .sha256，远端唯一路径 - git archive 产物 + DEPLOYED_COMMIT 清单，默认拒绝脏工作区 - 传输链 scp -> scp_run -> chunk 分片，每路传完远端验 sha，不符删掉重来 - 主机侧唯一脚本：先验 sha/清单，staging 解包 + 编译检查，换目录上线， work/ 原样保留，health + 上线 commit 不符即回滚并退出非零 - deploy.py 成功后独立 ssh 再读一次 live 清单二次核对 - CI：host-verify 加 deploy/tests；release.yml 只在手动/tag + production environment 审批时部署，主干推送绝不自动部署生产 - deploy.sh 标记为旧链路弃用 - health 的 commit 字段已在父提交 HA 阶段 0 里随 server.py 落地， 本分支整体包含它；commit 为空时只靠文件核对，门禁同样成立 测试：host 32、cloud 487（4 跳过）、deploy 9，全部通过； host_deploy.sh 仿真覆盖成功/坏 sha/清单不符/health commit 不符回滚。 生产实机部署未做，等确认后再用新链路推。 * HA stage 1: worktree_host + queued cross-host PR jobs --------- Co-authored-by: host <host@test.local>
This commit is contained in:
openqjandhost authored and GitHub committed
1 parent
455513dd33
commit
fc8e337d0f
14 files changed
+1639
-67
No files matched your search
+106
-8
@@ -129,6 +129,8 @@ from cloud.cost import CostStore, estimate_tokens, price_usd_micros, sum_usage
|
||||
from cloud.db import connect as _db_connect
|
||||
from cloud.logging_config import get_logger, log_event, setup_logging
|
||||
from cloud.models import ModelRouter, is_fallbackable, record_fallback
|
||||
from cloud.pr import PrSkipped, PrStepError
|
||||
from cloud.pr_exec import (claim_pr_job, execute_task_pr, finish_pr_job)
|
||||
from cloud.queue import budget_exceeded, ready_tasks
|
||||
from cloud.repos import RepoError, resolve_repo
|
||||
from cloud.sandbox import CloudExecutor, DockerExecutor, LocalExecutor
|
||||
@@ -245,7 +247,8 @@ class Worker:
|
||||
worker_id=None, capacity=1, fallback_models=None,
|
||||
tenant_isolation=False, tenants_root=None,
|
||||
tenant_template_home=None, tenant_template_repo=None,
|
||||
egress_proxy_port=0, claim_executor_only=False):
|
||||
egress_proxy_port=0, claim_executor_only=False,
|
||||
host_name=None, pr_jobs_only=False):
|
||||
self.db_path = str(db_path)
|
||||
self.host_py_path = host_py_path
|
||||
self.config_path = config_path
|
||||
@@ -338,6 +341,15 @@ class Worker:
|
||||
# 容器的 NULL 任务（那些任务在宿主上跑就丢了 Docker
|
||||
# 外层，违背“按任务二选一”的路由意图）。
|
||||
self.claim_executor_only = bool(claim_executor_only)
|
||||
# HA 阶段 1：本 worker 所在主机名。认领任务时写进
|
||||
# tasks.worktree_host，/pr 据此判断 worktree 在哪台主机。
|
||||
# 容器内 worker 的 socket.gethostname() 是容器名、不是宿主名，
|
||||
# 所以 autoscaler 必须显式传 --host-name（宿主名），见
|
||||
# autoscaler._worker_command；单机/测试默认本机名即可。
|
||||
self.host_name = host_name or socket.gethostname()
|
||||
# 只跑 pr 任务（HA 阶段 1 的跨主机 /pr 演练与专用 pr-runner
|
||||
# 用）：不认领普通任务，只处理本主机的 pr_jobs。
|
||||
self.pr_jobs_only = bool(pr_jobs_only)
|
||||
# 运维心跳：路径优先显式参数，其次环境变量，默认当前目录下
|
||||
# 的 worker-heartbeat.json；主机路径不在代码里硬编码，主机
|
||||
# 部署时用 --heartbeat-file 或环境变量传入（见 deploy/README）。
|
||||
@@ -467,25 +479,28 @@ class Worker:
|
||||
if executor is None:
|
||||
cur = conn.execute(
|
||||
"UPDATE tasks SET status = 'running',"
|
||||
" last_heartbeat = ?, updated_at = ?"
|
||||
" last_heartbeat = ?, updated_at = ?,"
|
||||
" worktree_host = ?"
|
||||
" WHERE id = ? AND status = 'queued'",
|
||||
(now, now, task_id),
|
||||
(now, now, self.host_name, task_id),
|
||||
)
|
||||
elif self.claim_executor_only:
|
||||
cur = conn.execute(
|
||||
"UPDATE tasks SET status = 'running',"
|
||||
" last_heartbeat = ?, updated_at = ?"
|
||||
" last_heartbeat = ?, updated_at = ?,"
|
||||
" worktree_host = ?"
|
||||
" WHERE id = ? AND status = 'queued'"
|
||||
" AND executor = ?",
|
||||
(now, now, task_id, executor),
|
||||
(now, now, self.host_name, task_id, executor),
|
||||
)
|
||||
else:
|
||||
cur = conn.execute(
|
||||
"UPDATE tasks SET status = 'running',"
|
||||
" last_heartbeat = ?, updated_at = ?"
|
||||
" last_heartbeat = ?, updated_at = ?,"
|
||||
" worktree_host = ?"
|
||||
" WHERE id = ? AND status = 'queued'"
|
||||
" AND (executor IS NULL OR executor = ?)",
|
||||
(now, now, task_id, executor),
|
||||
(now, now, self.host_name, task_id, executor),
|
||||
)
|
||||
conn.commit()
|
||||
return cur.rowcount == 1
|
||||
@@ -523,7 +538,8 @@ class Worker:
|
||||
"executor": "TEXT", "pr_url": "TEXT",
|
||||
"pr_number": "INTEGER", "goal": "TEXT",
|
||||
"verify_command": "TEXT",
|
||||
"goal_max_rounds": "INTEGER"}.items():
|
||||
"goal_max_rounds": "INTEGER",
|
||||
"worktree_host": "TEXT"}.items():
|
||||
if name not in cols:
|
||||
try:
|
||||
conn.execute(
|
||||
@@ -1651,6 +1667,73 @@ class Worker:
|
||||
"actual_model": actual_model,
|
||||
"cost_record_id": cost_id}
|
||||
|
||||
def _run_pr_jobs(self):
|
||||
"""跑本主机的 pr 任务（HA 阶段 1），返回处理结果列表。
|
||||
|
||||
只有 local worker 跑：host-worktrees/ 与 tenants/ 在主机上
|
||||
都可见，容器内 worker 看不到 host-worktrees/。每轮把本主机
|
||||
的 queued 任务按 CAS 认领跑完（pr 任务稀少，串行即可）；
|
||||
execute_task_pr 内部已写回任务 pr_url/pr_number、记审计、
|
||||
发 review-requested/merge-failed，这里只写回任务终态。
|
||||
跳过（PrSkipped，如任务已不是 merged）与落盘失败都记
|
||||
failed + error，任务本身保持 merged、可重调 /pr 再排队。
|
||||
"""
|
||||
if self._executor_name() != "local":
|
||||
return []
|
||||
results = []
|
||||
while True:
|
||||
conn = _db_connect(self.db_path)
|
||||
try:
|
||||
job = claim_pr_job(conn, self.host_name)
|
||||
finally:
|
||||
conn.close()
|
||||
if job is None:
|
||||
break
|
||||
job_id = job["id"]
|
||||
task_id = job["task_id"]
|
||||
try:
|
||||
result = execute_task_pr(
|
||||
self.db_path, task_id, "system")
|
||||
conn = _db_connect(self.db_path)
|
||||
try:
|
||||
finish_pr_job(conn, job_id, "completed",
|
||||
result["pr_url"], result["pr_number"])
|
||||
finally:
|
||||
conn.close()
|
||||
results.append({"pr_job_id": job_id, "task_id": task_id,
|
||||
"status": "completed",
|
||||
"pr_url": result["pr_url"]})
|
||||
log_event(self.logger, "pr_job_completed",
|
||||
task_id=task_id, status="completed")
|
||||
except PrSkipped as exc:
|
||||
err = "skipped: %s" % exc.reason
|
||||
conn = _db_connect(self.db_path)
|
||||
try:
|
||||
finish_pr_job(conn, job_id, "failed", error=err)
|
||||
finally:
|
||||
conn.close()
|
||||
results.append({"pr_job_id": job_id, "task_id": task_id,
|
||||
"status": "failed", "error": err})
|
||||
except PrStepError as exc:
|
||||
err = "%s: %s" % (exc.step, exc.message)
|
||||
conn = _db_connect(self.db_path)
|
||||
try:
|
||||
finish_pr_job(conn, job_id, "failed", error=err)
|
||||
finally:
|
||||
conn.close()
|
||||
results.append({"pr_job_id": job_id, "task_id": task_id,
|
||||
"status": "failed", "error": err})
|
||||
except Exception as exc: # noqa: BLE001 -- 任务失败不能炸循环
|
||||
err = str(exc)
|
||||
conn = _db_connect(self.db_path)
|
||||
try:
|
||||
finish_pr_job(conn, job_id, "failed", error=err)
|
||||
finally:
|
||||
conn.close()
|
||||
results.append({"pr_job_id": job_id, "task_id": task_id,
|
||||
"status": "failed", "error": err})
|
||||
return results
|
||||
|
||||
def run_all(self, max_tasks=None):
|
||||
"""循环认领执行，直到无就绪任务、预算超限或达到 max_tasks。
|
||||
|
||||
@@ -1668,6 +1751,11 @@ class Worker:
|
||||
停机信号（_loop_stop/stop_event）只阻止新认领，不会
|
||||
丢弃已提交的任务；排空等待由 run_loop 的 finally 负责。
|
||||
"""
|
||||
# pr 任务先跑（HA 阶段 1）：每轮先把本主机的 queued pr 任务
|
||||
# 跑完再认领普通任务；pr_jobs_only 时只跑 pr 任务。
|
||||
self._run_pr_jobs()
|
||||
if self.pr_jobs_only:
|
||||
return []
|
||||
if self.capacity <= 1:
|
||||
results = []
|
||||
while max_tasks is None or len(results) < max_tasks:
|
||||
@@ -1960,6 +2048,14 @@ def main(argv=None):
|
||||
help="worker 并发上限：同一 worker 内最多同时跑"
|
||||
"几个任务（默认 1，即旧串行行为；>1 时用"
|
||||
"线程池并行，认领仍是 CAS，成本写库持锁）")
|
||||
parser.add_argument("--host-name", default=None,
|
||||
help="本 worker 所在主机名（HA 阶段 1）：认领任务"
|
||||
"时写进 tasks.worktree_host，/pr 据此判断"
|
||||
"worktree 在哪台主机；容器内 worker 必须由"
|
||||
"autoscaler 显式传宿主名，不能用容器名")
|
||||
parser.add_argument("--pr-jobs-only", action="store_true",
|
||||
help="只跑 pr 任务（HA 阶段 1 的跨主机 /pr 演练"
|
||||
"与专用 pr-runner 用）：不认领普通任务")
|
||||
parser.add_argument("--docker-image", default="codex-app-server:latest")
|
||||
parser.add_argument("--docker-cpus", default="2")
|
||||
parser.add_argument("--docker-memory", default="2g")
|
||||
@@ -2040,6 +2136,8 @@ def main(argv=None):
|
||||
heartbeat_file=args.heartbeat_file,
|
||||
worker_id=args.worker_id,
|
||||
capacity=args.worker_capacity,
|
||||
host_name=args.host_name,
|
||||
pr_jobs_only=args.pr_jobs_only,
|
||||
tenant_isolation=args.tenant_isolation,
|
||||
tenants_root=args.tenants_root,
|
||||
tenant_template_home=args.tenant_template_home,
|
||||
|
||||
Reference in new issue
Block a user