HA 阶段 1：worktree_host + 跨主机 /pr 队列化 (#50)
* 部署自动化：唯一发布链路 deploy.py + host_deploy.sh，sha/commit 双重验真，失败回滚 旧链路三次栽过的坑：通用 tar 路径导致旧包误部署、传输不验 sha、 主机脚本多份漂移、health 通过不等于新代码上线。本单把四点变硬门禁： - 产物唯一命名 codex-app-server-<commit>.tar.gz + .sha256，远端唯一路径 - git archive 产物 + DEPLOYED_COMMIT 清单，默认拒绝脏工作区 - 传输链 scp -> scp_run -> chunk 分片，每路传完远端验 sha，不符删掉重来 - 主机侧唯一脚本：先验 sha/清单，staging 解包 + 编译检查，换目录上线， work/ 原样保留，health + 上线 commit 不符即回滚并退出非零 - deploy.py 成功后独立 ssh 再读一次 live 清单二次核对 - CI：host-verify 加 deploy/tests；release.yml 只在手动/tag + production environment 审批时部署，主干推送绝不自动部署生产 - deploy.sh 标记为旧链路弃用 - health 的 commit 字段已在父提交 HA 阶段 0 里随 server.py 落地， 本分支整体包含它；commit 为空时只靠文件核对，门禁同样成立 测试：host 32、cloud 487（4 跳过）、deploy 9，全部通过； host_deploy.sh 仿真覆盖成功/坏 sha/清单不符/health commit 不符回滚。 生产实机部署未做，等确认后再用新链路推。 * HA stage 1: worktree_host + queued cross-host PR jobs --------- Co-authored-by: host <host@test.local>
This commit is contained in:
openqjandhost authored and GitHub committed
1 parent
455513dd33
commit
fc8e337d0f
14 files changed
+1639
-67
No files matched your search
@@ -0,0 +1,325 @@
|
||||
#!/usr/bin/env python3
|
||||
"""统一发布入口：取代手工的 deploy.sh + scp_run + chunk_send + grep 验真。
|
||||
|
||||
为什么需要它（此前三次栽过的坑，见 deploy/README.md）：
|
||||
1. 通用 tar 路径 /tmp/codex-app-server-code.tar.gz：一次 scp 失败，
|
||||
旧包还躺在原路径，下一步手工跑主机脚本就把旧代码“部署成功”了。
|
||||
2. 传输不验 sha：scp 断在半路、chunk 首片写反，主机都可能拿到错包。
|
||||
3. 主机脚本多份：deploy.sh 内嵌版、deploy_*_host.sh 各自演进，行为漂移。
|
||||
4. health 通过不等于新代码上线：旧代码的 health 照样 ok=true。
|
||||
|
||||
本脚本把以上四点变成硬门禁，任何一步不符都退出非零、绝不打印成功：
|
||||
- 产物唯一命名：codex-app-server-<commit>.tar.gz + .sha256，
|
||||
远端路径 /tmp/codex-app-server-releases/<commit>.tar.gz，旧包无法冒充新包。
|
||||
- 产物来自 git archive（已提交的 HEAD），默认拒绝工作区脏文件；
|
||||
tar 根目录带 DEPLOYED_COMMIT 清单，主机侧先验清单再动代码。
|
||||
- 传输链自动 fallback：scp（SSH_PROXY_CMD）-> ssh-helpers/scp_run.py
|
||||
-> ssh-helpers/chunk_send.py（base64 分片），每种方式传完都先在远端
|
||||
跑 sha256sum 比对，不符就删掉该次产物再试下一种，绝不把错包交给主机脚本。
|
||||
- 主机侧只用 deploy/host_deploy.sh 这一份（VM 侧随包一起传过去，
|
||||
不依赖主机旧代码里的版本），它做 staging 换目录 + 回滚。
|
||||
- 上线证据双重核对：主机脚本打印的 RELEASE_OK 里的 commit，
|
||||
再加 VM 侧独立 ssh cat live 的 DEPLOYED_COMMIT，两者都必须等于期望 commit。
|
||||
|
||||
用法：
|
||||
python3 deploy/deploy.py <host> [user] [--ref HEAD] [--dry-run]
|
||||
python3 deploy/deploy.py <host> --build-only [--out-dir DIR]
|
||||
python3 deploy/deploy.py <host> --tarball <path> --commit <sha>
|
||||
python3 deploy/deploy.py <host> --skip-tests # CI 里测试已跑过时用
|
||||
|
||||
密钥规则：本脚本绝不接受密码参数、不读任何 env 文件、不打印任何密钥。
|
||||
SSH 密码只由调用方已有的通道处理（key，或 ~/workspace/ssh-helpers 的
|
||||
pty helper，密码只走 SSH_PASSWORD 环境变量）；GitHub Actions 里走 key。
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tarfile
|
||||
import tempfile
|
||||
from datetime import datetime, timezone
|
||||
|
||||
REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
HOST_SCRIPT = os.path.join(REPO_ROOT, "deploy", "host_deploy.sh")
|
||||
HELPER_DIR = os.path.expanduser("~/workspace/ssh-helpers")
|
||||
REMOTE_RELEASES_DIR = "/tmp/codex-app-server-releases"
|
||||
|
||||
|
||||
def run(cmd, cwd=None, check=True):
|
||||
"""跑命令并返回 stdout；失败时把 stderr 透传后按 check 决定是否退出。"""
|
||||
r = subprocess.run(cmd, cwd=cwd, capture_output=True, text=True)
|
||||
if check and r.returncode != 0:
|
||||
print("命令失败：%s" % " ".join(cmd), file=sys.stderr)
|
||||
print((r.stdout + r.stderr)[-2000:], file=sys.stderr)
|
||||
sys.exit(1)
|
||||
return r
|
||||
|
||||
|
||||
def git_commit(ref):
|
||||
return run(["git", "rev-parse", ref], cwd=REPO_ROOT).stdout.strip()
|
||||
|
||||
|
||||
def check_clean_tree(allow_dirty):
|
||||
"""已提交内容与工作区不一致时，默认拒绝；--allow-dirty 只告警，
|
||||
且部署的仍是 HEAD 的 archive，脏文件不会被偷偷带上去。"""
|
||||
diff = run(["git", "diff", "--quiet"], cwd=REPO_ROOT, check=False)
|
||||
cached = run(["git", "diff", "--cached", "--quiet"], cwd=REPO_ROOT, check=False)
|
||||
if diff.returncode != 0 or cached.returncode != 0:
|
||||
if not allow_dirty:
|
||||
print("工作区有未提交改动，先提交再部署（或显式 --allow-dirty，"
|
||||
"但部署的仍是 HEAD，不含脏文件）。", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
print("警告：工作区有脏文件，部署的仍是 HEAD 的 archive，"
|
||||
"脏文件不会上线。")
|
||||
|
||||
|
||||
def sha256_of(path):
|
||||
h = hashlib.sha256()
|
||||
with open(path, "rb") as f:
|
||||
for chunk in iter(lambda: f.read(1024 * 1024), b""):
|
||||
h.update(chunk)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def release_tarball_name(commit):
|
||||
return "codex-app-server-%s.tar.gz" % commit
|
||||
|
||||
|
||||
def build_release(ref, out_dir):
|
||||
"""从 git archive 建发布包，返回 (tarball_path, sha256, commit)。"""
|
||||
commit = git_commit(ref)
|
||||
os.makedirs(out_dir, exist_ok=True)
|
||||
tarball = os.path.join(out_dir, release_tarball_name(commit))
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
# git archive 输出是二进制 tar 流，直接写文件，不经过文本解码。
|
||||
with open(os.path.join(tmp, "archive.tar"), "wb") as f:
|
||||
r = subprocess.run(["git", "archive", ref], cwd=REPO_ROOT, stdout=f)
|
||||
if r.returncode != 0:
|
||||
print("git archive 失败：%s" % ref, file=sys.stderr)
|
||||
sys.exit(1)
|
||||
with tarfile.open(os.path.join(tmp, "archive.tar")) as tf:
|
||||
tf.extractall(tmp, filter="data")
|
||||
with open(os.path.join(tmp, "DEPLOYED_COMMIT"), "w") as f:
|
||||
f.write(commit + "\n")
|
||||
with open(os.path.join(tmp, "RELEASE.json"), "w") as f:
|
||||
json.dump({
|
||||
"commit": commit, "ref": ref,
|
||||
"built_at": datetime.now(timezone.utc).isoformat(),
|
||||
}, f, indent=2, sort_keys=True)
|
||||
f.write("\n")
|
||||
with tarfile.open(tarball, "w:gz", format=tarfile.PAX_FORMAT) as tf:
|
||||
for root, dirs, files in os.walk(tmp):
|
||||
dirs.sort()
|
||||
for name in sorted(files):
|
||||
full = os.path.join(root, name)
|
||||
arc = os.path.relpath(full, tmp)
|
||||
if arc in ("archive.tar",):
|
||||
continue
|
||||
tf.add(full, arcname=arc)
|
||||
sha = sha256_of(tarball)
|
||||
with open(tarball + ".sha256", "w") as f:
|
||||
f.write("%s %s\n" % (sha, os.path.basename(tarball)))
|
||||
return tarball, sha, commit
|
||||
|
||||
|
||||
def tarball_commit(tarball):
|
||||
with tarfile.open(tarball, "r:gz") as tf:
|
||||
try:
|
||||
member = tf.getmember("DEPLOYED_COMMIT")
|
||||
except KeyError:
|
||||
print("发布包里没有 DEPLOYED_COMMIT，拒绝部署：%s" % tarball,
|
||||
file=sys.stderr)
|
||||
sys.exit(2)
|
||||
data = tf.extractfile(member).read().decode().strip()
|
||||
return data
|
||||
|
||||
|
||||
def parse_release_ok(output, expected_commit):
|
||||
"""主机脚本成功输出里必须有 RELEASE_OK 且 commit 等于期望值。"""
|
||||
for line in output.splitlines():
|
||||
if line.startswith("RELEASE_OK"):
|
||||
fields = dict(
|
||||
part.split("=", 1) for part in line.split()[1:]
|
||||
if "=" in part)
|
||||
return fields.get("commit") == expected_commit
|
||||
return False
|
||||
|
||||
|
||||
# ---------------- 传输：ssh / scp 抽象 ----------------
|
||||
|
||||
def helper_path(name):
|
||||
path = os.path.join(HELPER_DIR, name)
|
||||
return path if os.path.isfile(path) else None
|
||||
|
||||
|
||||
def ssh_cmd(host, user, remote_cmd):
|
||||
"""返回一条 ssh 命令。优先用已验证稳定的 helper，否则用
|
||||
SSH_PROXY_CMD 环境变量，否则直连。绝不把密码拼进命令。"""
|
||||
helper = helper_path("ssh_run.py")
|
||||
if helper:
|
||||
return ["python3", helper, "ssh", host, user, remote_cmd]
|
||||
proxy = os.environ.get("SSH_PROXY_CMD")
|
||||
base = ["ssh", "-o", "StrictHostKeyChecking=no",
|
||||
"-o", "UserKnownHostsFile=/dev/null"]
|
||||
if proxy:
|
||||
base += ["-o", "ProxyCommand %s" % proxy]
|
||||
return base + ["%s@%s" % (user, host), remote_cmd]
|
||||
|
||||
|
||||
def scp_cmd(host, user, local, remote):
|
||||
helper = helper_path("scp_run.py")
|
||||
if helper:
|
||||
# helper 自带 egress proxy + 密码 pty，是此前实测最稳的一路。
|
||||
return ["python3", helper, "scp", host, user, local, remote]
|
||||
proxy = os.environ.get("SSH_PROXY_CMD")
|
||||
base = ["scp", "-o", "StrictHostKeyChecking=no",
|
||||
"-o", "UserKnownHostsFile=/dev/null"]
|
||||
if proxy:
|
||||
base += ["-o", "ProxyCommand %s" % proxy]
|
||||
return base + [local, "%s@%s:%s" % (user, host, remote)]
|
||||
|
||||
|
||||
def remote_sha(host, user, remote_path):
|
||||
r = subprocess.run(ssh_cmd(host, user, "sha256sum %s" % remote_path),
|
||||
capture_output=True, text=True)
|
||||
if r.returncode != 0:
|
||||
return None
|
||||
return r.stdout.split()[0] if r.stdout.split() else None
|
||||
|
||||
|
||||
def remote_rm(host, user, remote_path):
|
||||
subprocess.run(ssh_cmd(host, user, "rm -f %s" % remote_path),
|
||||
capture_output=True, text=True)
|
||||
|
||||
|
||||
def transfer(host, user, local, remote, expected_sha):
|
||||
"""传输链：scp -> helper scp -> chunk 分片。每路传完都验远端 sha，
|
||||
不符删掉重来；全部失败才退出，绝不把错包交给主机脚本。"""
|
||||
attempts = []
|
||||
# 1. scp（含 helper 路由，由 scp_cmd 内部决定）
|
||||
attempts.append(("scp", scp_cmd(host, user, local, remote)))
|
||||
chunk = helper_path("chunk_send.py")
|
||||
if chunk and os.environ.get("SSH_PASSWORD"):
|
||||
remote_b64 = remote + ".b64"
|
||||
attempts.append(("chunk", None)) # 特殊处理，见下
|
||||
for name, cmd in attempts:
|
||||
print("传输尝试：%s -> %s" % (name, remote))
|
||||
if name == "chunk":
|
||||
r = subprocess.run(
|
||||
["python3", chunk, local, remote_b64, host, user],
|
||||
capture_output=True, text=True)
|
||||
if r.returncode != 0:
|
||||
print("chunk 传输失败，换下一路。", file=sys.stderr)
|
||||
continue
|
||||
r = subprocess.run(
|
||||
ssh_cmd(host, user,
|
||||
"base64 -d %s > %s && sha256sum %s"
|
||||
% (remote_b64, remote, remote)),
|
||||
capture_output=True, text=True)
|
||||
if r.returncode != 0:
|
||||
print("chunk 解码失败，换下一路。", file=sys.stderr)
|
||||
continue
|
||||
else:
|
||||
r = subprocess.run(cmd, capture_output=True, text=True)
|
||||
if r.returncode != 0:
|
||||
print("scp 失败（%s），换下一路。" % (r.stderr or "").strip()[-200:],
|
||||
file=sys.stderr)
|
||||
continue
|
||||
actual = remote_sha(host, user, remote)
|
||||
if actual == expected_sha:
|
||||
print("传输验真通过：%s sha=%s" % (name, actual))
|
||||
return
|
||||
print("远端 sha 不符（expected=%s actual=%s），删掉该次产物，换下一路。"
|
||||
% (expected_sha, actual), file=sys.stderr)
|
||||
remote_rm(host, user, remote)
|
||||
print("所有传输方式都失败或验真不通过，部署未开始（旧代码不动）。",
|
||||
file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def run_tests():
|
||||
for suite in ("tests", "cloud/tests"):
|
||||
run(["python3", "-m", "unittest", "discover", "-s", suite],
|
||||
cwd=REPO_ROOT)
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
parser = argparse.ArgumentParser(description="codex-app-server 统一发布入口")
|
||||
parser.add_argument("host", nargs="?", help="部署目标主机")
|
||||
parser.add_argument("user", nargs="?", default="root", help="SSH 用户")
|
||||
parser.add_argument("--ref", default="HEAD")
|
||||
parser.add_argument("--tarball", help="直接部署已有发布包（CI 产物）")
|
||||
parser.add_argument("--commit", help="配合 --tarball 指定期望 commit")
|
||||
parser.add_argument("--out-dir", default=os.path.join(
|
||||
REPO_ROOT, "work", "external-install", "releases"))
|
||||
parser.add_argument("--allow-dirty", action="store_true")
|
||||
parser.add_argument("--skip-tests", action="store_true")
|
||||
parser.add_argument("--build-only", action="store_true")
|
||||
parser.add_argument("--dry-run", action="store_true")
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
if args.tarball:
|
||||
commit = args.commit or tarball_commit(args.tarball)
|
||||
tarball, sha = args.tarball, sha256_of(args.tarball)
|
||||
if tarball_commit(tarball) != commit:
|
||||
print("发布包内 DEPLOYED_COMMIT 与 --commit 不符，拒绝部署。",
|
||||
file=sys.stderr)
|
||||
sys.exit(2)
|
||||
else:
|
||||
if not args.build_only and not args.dry_run and not args.host:
|
||||
parser.error("需要 host，或用 --build-only / --dry-run")
|
||||
check_clean_tree(args.allow_dirty)
|
||||
if not args.skip_tests:
|
||||
run_tests()
|
||||
tarball, sha, commit = build_release(args.ref, args.out_dir)
|
||||
print("发布包：%s" % tarball)
|
||||
print("commit=%s sha256=%s" % (commit, sha))
|
||||
|
||||
if args.build_only:
|
||||
print("仅构建，不传输、不部署。")
|
||||
return 0
|
||||
if args.dry_run:
|
||||
print("[dry-run] 将传输 %s 到 %s@%s:%s/%s（唯一路径，传完验 sha）"
|
||||
% (tarball, args.user, args.host, REMOTE_RELEASES_DIR,
|
||||
os.path.basename(tarball)))
|
||||
print("[dry-run] 将传主机脚本 %s 并执行："
|
||||
"sh host_deploy.sh <tarball> <sha> <commit>，"
|
||||
"成功须打印 RELEASE_OK 且 commit 一致，否则回滚。" % HOST_SCRIPT)
|
||||
return 0
|
||||
|
||||
host, user = args.host, args.user
|
||||
remote_tar = "%s/%s" % (REMOTE_RELEASES_DIR, os.path.basename(tarball))
|
||||
remote_script = "%s/host_deploy-%s.sh" % (REMOTE_RELEASES_DIR, commit)
|
||||
run(ssh_cmd(host, user, "mkdir -p %s" % REMOTE_RELEASES_DIR), check=False)
|
||||
transfer(host, user, tarball, remote_tar, sha)
|
||||
transfer(host, user, HOST_SCRIPT, remote_script, sha256_of(HOST_SCRIPT))
|
||||
print("执行主机部署脚本（唯一脚本，staging 换目录 + 失败回滚）…")
|
||||
r = subprocess.run(
|
||||
ssh_cmd(host, user,
|
||||
"sh %s %s %s %s" % (remote_script, remote_tar, sha, commit)),
|
||||
capture_output=True, text=True)
|
||||
print(r.stdout)
|
||||
if r.stderr:
|
||||
print(r.stderr, file=sys.stderr)
|
||||
if r.returncode != 0 or not parse_release_ok(r.stdout, commit):
|
||||
print("主机部署未返回有效的 RELEASE_OK，部署失败（主机侧已回滚或未动）。",
|
||||
file=sys.stderr)
|
||||
sys.exit(1)
|
||||
# 独立二次核对：不只信主机脚本的打印，再 ssh 读一次 live 清单。
|
||||
live = subprocess.run(
|
||||
ssh_cmd(host, user, "cat /opt/codex-app-server/DEPLOYED_COMMIT"),
|
||||
capture_output=True, text=True)
|
||||
if live.stdout.strip() != commit:
|
||||
print("二次核对失败：live DEPLOYED_COMMIT=%s，期望 %s。"
|
||||
"请立即用 deploy/rollback.sh 检查，不要再推新版。"
|
||||
% (live.stdout.strip(), commit), file=sys.stderr)
|
||||
sys.exit(1)
|
||||
print("部署成功且二次核对通过：commit=%s sha256=%s" % (commit, sha))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in new issue
Block a user