host
420a7fecfd
背景：2026-10-27 另一条线给 deploy.py 打的密钥补丁只放在 env-console 工作区未提交，切分支后丢失；部署一度只能靠它 + --allow-dirty。本批把逻辑收进 main，并加一处加固。 分发：有 SSH_PASSWORD 时走 helper（pty 喂密码，实测最稳的一 路）；无密码时 helper 不可用（会卡在密码提示），改走密钥直连： ProxyCommand 显式环境变量优先、其次 helper 目录的 proxy_tunnel.py（存在才用），密钥按 CODEX_DEPLOY_KEY → /root/.ssh/id_codex_deploy → ~/.ssh/id_codex_deploy 顺序显式传 -i——不能只依赖 /root/.ssh/config 的 Host 条目，那个目录会在 VM 回收时被清空（2026-10-27 05:04 实测），而 ~/.ssh 的那份 持久。两处都没有时不硬编造，靠 ssh 配置或 agent。 测试：deploy/tests 新增 TestSshScpCmd 5 个（有密码走 helper、 无密码绝不用 helper、无 helper 无代理时直连、显式代理优先、 显式密钥优先）；全量 deploy 19、host 62、cloud 598 全过。
377 lines
16 KiB
Python
377 lines
16 KiB
Python
#!/usr/bin/env python3
|
||
"""统一发布入口：取代手工的 deploy.sh + scp_run + chunk_send + grep 验真。
|
||
|
||
为什么需要它（此前三次栽过的坑，见 deploy/README.md）：
|
||
1. 通用 tar 路径 /tmp/codex-app-server-code.tar.gz：一次 scp 失败，
|
||
旧包还躺在原路径，下一步手工跑主机脚本就把旧代码“部署成功”了。
|
||
2. 传输不验 sha：scp 断在半路、chunk 首片写反，主机都可能拿到错包。
|
||
3. 主机脚本多份：deploy.sh 内嵌版、deploy_*_host.sh 各自演进，行为漂移。
|
||
4. health 通过不等于新代码上线：旧代码的 health 照样 ok=true。
|
||
|
||
本脚本把以上四点变成硬门禁，任何一步不符都退出非零、绝不打印成功：
|
||
- 产物唯一命名：codex-app-server-<commit>.tar.gz + .sha256，
|
||
远端路径 /tmp/codex-app-server-releases/<commit>.tar.gz，旧包无法冒充新包。
|
||
- 产物来自 git archive（已提交的 HEAD），默认拒绝工作区脏文件；
|
||
tar 根目录带 DEPLOYED_COMMIT 清单，主机侧先验清单再动代码。
|
||
- 传输链自动 fallback：scp（SSH_PROXY_CMD）-> ssh-helpers/scp_run.py
|
||
-> ssh-helpers/chunk_send.py（base64 分片），每种方式传完都先在远端
|
||
跑 sha256sum 比对，不符就删掉该次产物再试下一种，绝不把错包交给主机脚本。
|
||
- 主机侧只用 deploy/host_deploy.sh 这一份（VM 侧随包一起传过去，
|
||
不依赖主机旧代码里的版本），它做 staging 换目录 + 回滚。
|
||
- 上线证据双重核对：主机脚本打印的 RELEASE_OK 里的 commit，
|
||
再加 VM 侧独立 ssh cat live 的 DEPLOYED_COMMIT，两者都必须等于期望 commit。
|
||
|
||
用法：
|
||
python3 deploy/deploy.py <host> [user] [--ref HEAD] [--dry-run]
|
||
python3 deploy/deploy.py <host> --build-only [--out-dir DIR]
|
||
python3 deploy/deploy.py <host> --tarball <path> --commit <sha>
|
||
python3 deploy/deploy.py <host> --skip-tests # CI 里测试已跑过时用
|
||
|
||
密钥规则：本脚本绝不接受密码参数、不读任何 env 文件、不打印任何密钥。
|
||
SSH 密码只由调用方已有的通道处理（key，或 ~/workspace/ssh-helpers 的
|
||
pty helper，密码只走 SSH_PASSWORD 环境变量）；GitHub Actions 里走 key。
|
||
"""
|
||
import argparse
|
||
import hashlib
|
||
import json
|
||
import os
|
||
import re
|
||
import shutil
|
||
import subprocess
|
||
import sys
|
||
import tarfile
|
||
import tempfile
|
||
from datetime import datetime, timezone
|
||
|
||
REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||
HOST_SCRIPT = os.path.join(REPO_ROOT, "deploy", "host_deploy.sh")
|
||
HELPER_DIR = os.path.expanduser("~/workspace/ssh-helpers")
|
||
REMOTE_RELEASES_DIR = "/tmp/codex-app-server-releases"
|
||
|
||
|
||
def run(cmd, cwd=None, check=True):
|
||
"""跑命令并返回 stdout；失败时把 stderr 透传后按 check 决定是否退出。"""
|
||
r = subprocess.run(cmd, cwd=cwd, capture_output=True, text=True)
|
||
if check and r.returncode != 0:
|
||
print("命令失败：%s" % " ".join(cmd), file=sys.stderr)
|
||
print((r.stdout + r.stderr)[-2000:], file=sys.stderr)
|
||
sys.exit(1)
|
||
return r
|
||
|
||
|
||
def git_commit(ref):
|
||
return run(["git", "rev-parse", ref], cwd=REPO_ROOT).stdout.strip()
|
||
|
||
|
||
def check_clean_tree(allow_dirty):
|
||
"""已提交内容与工作区不一致时，默认拒绝；--allow-dirty 只告警，
|
||
且部署的仍是 HEAD 的 archive，脏文件不会被偷偷带上去。"""
|
||
diff = run(["git", "diff", "--quiet"], cwd=REPO_ROOT, check=False)
|
||
cached = run(["git", "diff", "--cached", "--quiet"], cwd=REPO_ROOT, check=False)
|
||
if diff.returncode != 0 or cached.returncode != 0:
|
||
if not allow_dirty:
|
||
print("工作区有未提交改动，先提交再部署（或显式 --allow-dirty，"
|
||
"但部署的仍是 HEAD，不含脏文件）。", file=sys.stderr)
|
||
sys.exit(2)
|
||
print("警告：工作区有脏文件，部署的仍是 HEAD 的 archive，"
|
||
"脏文件不会上线。")
|
||
|
||
|
||
def sha256_of(path):
|
||
h = hashlib.sha256()
|
||
with open(path, "rb") as f:
|
||
for chunk in iter(lambda: f.read(1024 * 1024), b""):
|
||
h.update(chunk)
|
||
return h.hexdigest()
|
||
|
||
|
||
def release_tarball_name(commit):
|
||
return "codex-app-server-%s.tar.gz" % commit
|
||
|
||
|
||
def build_release(ref, out_dir):
|
||
"""从 git archive 建发布包，返回 (tarball_path, sha256, commit)。"""
|
||
commit = git_commit(ref)
|
||
os.makedirs(out_dir, exist_ok=True)
|
||
tarball = os.path.join(out_dir, release_tarball_name(commit))
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
# git archive 输出是二进制 tar 流，直接写文件，不经过文本解码。
|
||
with open(os.path.join(tmp, "archive.tar"), "wb") as f:
|
||
r = subprocess.run(["git", "archive", ref], cwd=REPO_ROOT, stdout=f)
|
||
if r.returncode != 0:
|
||
print("git archive 失败：%s" % ref, file=sys.stderr)
|
||
sys.exit(1)
|
||
with tarfile.open(os.path.join(tmp, "archive.tar")) as tf:
|
||
tf.extractall(tmp, filter="data")
|
||
with open(os.path.join(tmp, "DEPLOYED_COMMIT"), "w") as f:
|
||
f.write(commit + "\n")
|
||
with open(os.path.join(tmp, "RELEASE.json"), "w") as f:
|
||
json.dump({
|
||
"commit": commit, "ref": ref,
|
||
"built_at": datetime.now(timezone.utc).isoformat(),
|
||
}, f, indent=2, sort_keys=True)
|
||
f.write("\n")
|
||
with tarfile.open(tarball, "w:gz", format=tarfile.PAX_FORMAT) as tf:
|
||
for root, dirs, files in os.walk(tmp):
|
||
dirs.sort()
|
||
for name in sorted(files):
|
||
full = os.path.join(root, name)
|
||
arc = os.path.relpath(full, tmp)
|
||
if arc in ("archive.tar",):
|
||
continue
|
||
tf.add(full, arcname=arc)
|
||
sha = sha256_of(tarball)
|
||
with open(tarball + ".sha256", "w") as f:
|
||
f.write("%s %s\n" % (sha, os.path.basename(tarball)))
|
||
return tarball, sha, commit
|
||
|
||
|
||
def tarball_commit(tarball):
|
||
with tarfile.open(tarball, "r:gz") as tf:
|
||
try:
|
||
member = tf.getmember("DEPLOYED_COMMIT")
|
||
except KeyError:
|
||
print("发布包里没有 DEPLOYED_COMMIT，拒绝部署：%s" % tarball,
|
||
file=sys.stderr)
|
||
sys.exit(2)
|
||
data = tf.extractfile(member).read().decode().strip()
|
||
return data
|
||
|
||
|
||
def parse_release_ok(output, expected_commit):
|
||
"""主机脚本成功输出里必须有 RELEASE_OK 且 commit 等于期望值。"""
|
||
for line in output.splitlines():
|
||
if line.startswith("RELEASE_OK"):
|
||
fields = dict(
|
||
part.split("=", 1) for part in line.split()[1:]
|
||
if "=" in part)
|
||
return fields.get("commit") == expected_commit
|
||
return False
|
||
|
||
|
||
# ---------------- 传输：ssh / scp 抽象 ----------------
|
||
|
||
def helper_path(name):
|
||
path = os.path.join(HELPER_DIR, name)
|
||
return path if os.path.isfile(path) else None
|
||
|
||
|
||
def _deploy_key():
|
||
"""无密码时的部署密钥路径：显式环境变量优先，其次两处惯常
|
||
位置。/root/.ssh 的那份会在 VM 回收时被清空，~/.ssh 的那份
|
||
持久；两处都没有时返回 None（此时靠 ssh 配置或 agent，不
|
||
硬编造路径）。这里只返回路径，不读密钥内容。"""
|
||
env_key = os.environ.get("CODEX_DEPLOY_KEY")
|
||
if env_key:
|
||
return env_key
|
||
for cand in ("/root/.ssh/id_codex_deploy",
|
||
os.path.expanduser("~/.ssh/id_codex_deploy")):
|
||
if os.path.isfile(cand):
|
||
return cand
|
||
return None
|
||
|
||
|
||
def _proxy_cmd():
|
||
"""无密码时的 ProxyCommand：显式环境变量优先，其次 helper
|
||
目录的 proxy_tunnel.py（存在才用）；都没有时返回 None（直
|
||
连，不把不存在的脚本写进命令）。"""
|
||
proxy = os.environ.get("SSH_PROXY_CMD")
|
||
if proxy:
|
||
return proxy
|
||
default_proxy = os.path.join(HELPER_DIR, "proxy_tunnel.py")
|
||
if os.path.isfile(default_proxy):
|
||
return "python3 %s %%h %%p" % default_proxy
|
||
return None
|
||
|
||
|
||
def ssh_cmd(host, user, remote_cmd):
|
||
"""返回一条 ssh 命令。有 SSH_PASSWORD 时优先用已验证稳定的
|
||
helper（pty 喂密码）；无密码时走密钥直连：ProxyCommand 按
|
||
_proxy_cmd()，密钥按 _deploy_key() 显式传 -i（不能只依赖
|
||
/root/.ssh/config 的 Host 条目，那个目录会在 VM 回收时被
|
||
清空）。绝不把密码拼进命令。"""
|
||
helper = helper_path("ssh_run.py")
|
||
if helper and os.environ.get("SSH_PASSWORD"):
|
||
return ["python3", helper, "ssh", host, user, remote_cmd]
|
||
base = ["ssh", "-o", "StrictHostKeyChecking=no",
|
||
"-o", "UserKnownHostsFile=/dev/null"]
|
||
proxy = _proxy_cmd()
|
||
if proxy:
|
||
base += ["-o", "ProxyCommand %s" % proxy]
|
||
key = _deploy_key()
|
||
if key:
|
||
base += ["-i", key]
|
||
return base + ["%s@%s" % (user, host), remote_cmd]
|
||
|
||
|
||
def scp_cmd(host, user, local, remote):
|
||
helper = helper_path("scp_run.py")
|
||
if helper and os.environ.get("SSH_PASSWORD"):
|
||
# helper 自带 egress proxy + 密码 pty，是此前实测最稳的一路。
|
||
return ["python3", helper, "scp", host, user, local, remote]
|
||
base = ["scp", "-o", "StrictHostKeyChecking=no",
|
||
"-o", "UserKnownHostsFile=/dev/null"]
|
||
proxy = _proxy_cmd()
|
||
if proxy:
|
||
base += ["-o", "ProxyCommand %s" % proxy]
|
||
key = _deploy_key()
|
||
if key:
|
||
base += ["-i", key]
|
||
return base + [local, "%s@%s:%s" % (user, host, remote)]
|
||
|
||
|
||
def remote_sha(host, user, remote_path):
|
||
"""远端 sha256sum 的输出里取 64 位 hex。不能取 split()[0]：经 pty
|
||
的 ssh helper 会把 stderr 的 Warning（例如 Permanently added to
|
||
the list of known hosts）合并进 stdout 放在 sha 行前面，首 token
|
||
于是变成 Warning:，导致传输明明成功却被判验真失败（2026-10-27
|
||
新链路首推实测）。"""
|
||
r = subprocess.run(ssh_cmd(host, user, "sha256sum %s" % remote_path),
|
||
capture_output=True, text=True)
|
||
if r.returncode != 0:
|
||
return None
|
||
m = re.search(r"\b[0-9a-f]{64}\b", r.stdout)
|
||
return m.group(0) if m else None
|
||
|
||
|
||
def live_commit(text):
|
||
"""从 ssh 输出里取 40 位 commit。理由同 remote_sha：helper 的
|
||
Warning 行会混进 stdout，直接 strip() 比对必失败。"""
|
||
m = re.search(r"\b[0-9a-f]{40}\b", text)
|
||
return m.group(0) if m else None
|
||
|
||
|
||
def remote_rm(host, user, remote_path):
|
||
subprocess.run(ssh_cmd(host, user, "rm -f %s" % remote_path),
|
||
capture_output=True, text=True)
|
||
|
||
|
||
def transfer(host, user, local, remote, expected_sha):
|
||
"""传输链：scp -> helper scp -> chunk 分片。每路传完都验远端 sha，
|
||
不符删掉重来；全部失败才退出，绝不把错包交给主机脚本。"""
|
||
attempts = []
|
||
# 1. scp（含 helper 路由，由 scp_cmd 内部决定）
|
||
attempts.append(("scp", scp_cmd(host, user, local, remote)))
|
||
chunk = helper_path("chunk_send.py")
|
||
if chunk and os.environ.get("SSH_PASSWORD"):
|
||
remote_b64 = remote + ".b64"
|
||
attempts.append(("chunk", None)) # 特殊处理，见下
|
||
for name, cmd in attempts:
|
||
print("传输尝试：%s -> %s" % (name, remote))
|
||
if name == "chunk":
|
||
r = subprocess.run(
|
||
["python3", chunk, local, remote_b64, host, user],
|
||
capture_output=True, text=True)
|
||
if r.returncode != 0:
|
||
print("chunk 传输失败，换下一路。", file=sys.stderr)
|
||
continue
|
||
r = subprocess.run(
|
||
ssh_cmd(host, user,
|
||
"base64 -d %s > %s && sha256sum %s"
|
||
% (remote_b64, remote, remote)),
|
||
capture_output=True, text=True)
|
||
if r.returncode != 0:
|
||
print("chunk 解码失败，换下一路。", file=sys.stderr)
|
||
continue
|
||
else:
|
||
r = subprocess.run(cmd, capture_output=True, text=True)
|
||
if r.returncode != 0:
|
||
print("scp 失败（%s），换下一路。" % (r.stderr or "").strip()[-200:],
|
||
file=sys.stderr)
|
||
continue
|
||
actual = remote_sha(host, user, remote)
|
||
if actual == expected_sha:
|
||
print("传输验真通过：%s sha=%s" % (name, actual))
|
||
return
|
||
print("远端 sha 不符（expected=%s actual=%s），删掉该次产物，换下一路。"
|
||
% (expected_sha, actual), file=sys.stderr)
|
||
remote_rm(host, user, remote)
|
||
print("所有传输方式都失败或验真不通过，部署未开始（旧代码不动）。",
|
||
file=sys.stderr)
|
||
sys.exit(1)
|
||
|
||
|
||
def run_tests():
|
||
for suite in ("tests", "cloud/tests"):
|
||
run(["python3", "-m", "unittest", "discover", "-s", suite],
|
||
cwd=REPO_ROOT)
|
||
|
||
|
||
def main(argv=None):
|
||
parser = argparse.ArgumentParser(description="codex-app-server 统一发布入口")
|
||
parser.add_argument("host", nargs="?", help="部署目标主机")
|
||
parser.add_argument("user", nargs="?", default="root", help="SSH 用户")
|
||
parser.add_argument("--ref", default="HEAD")
|
||
parser.add_argument("--tarball", help="直接部署已有发布包（CI 产物）")
|
||
parser.add_argument("--commit", help="配合 --tarball 指定期望 commit")
|
||
parser.add_argument("--out-dir", default=os.path.join(
|
||
REPO_ROOT, "work", "external-install", "releases"))
|
||
parser.add_argument("--allow-dirty", action="store_true")
|
||
parser.add_argument("--skip-tests", action="store_true")
|
||
parser.add_argument("--build-only", action="store_true")
|
||
parser.add_argument("--dry-run", action="store_true")
|
||
args = parser.parse_args(argv)
|
||
|
||
if args.tarball:
|
||
commit = args.commit or tarball_commit(args.tarball)
|
||
tarball, sha = args.tarball, sha256_of(args.tarball)
|
||
if tarball_commit(tarball) != commit:
|
||
print("发布包内 DEPLOYED_COMMIT 与 --commit 不符，拒绝部署。",
|
||
file=sys.stderr)
|
||
sys.exit(2)
|
||
else:
|
||
if not args.build_only and not args.dry_run and not args.host:
|
||
parser.error("需要 host，或用 --build-only / --dry-run")
|
||
check_clean_tree(args.allow_dirty)
|
||
if not args.skip_tests:
|
||
run_tests()
|
||
tarball, sha, commit = build_release(args.ref, args.out_dir)
|
||
print("发布包：%s" % tarball)
|
||
print("commit=%s sha256=%s" % (commit, sha))
|
||
|
||
if args.build_only:
|
||
print("仅构建，不传输、不部署。")
|
||
return 0
|
||
if args.dry_run:
|
||
print("[dry-run] 将传输 %s 到 %s@%s:%s/%s（唯一路径，传完验 sha）"
|
||
% (tarball, args.user, args.host, REMOTE_RELEASES_DIR,
|
||
os.path.basename(tarball)))
|
||
print("[dry-run] 将传主机脚本 %s 并执行："
|
||
"sh host_deploy.sh <tarball> <sha> <commit>，"
|
||
"成功须打印 RELEASE_OK 且 commit 一致，否则回滚。" % HOST_SCRIPT)
|
||
return 0
|
||
|
||
host, user = args.host, args.user
|
||
remote_tar = "%s/%s" % (REMOTE_RELEASES_DIR, os.path.basename(tarball))
|
||
remote_script = "%s/host_deploy-%s.sh" % (REMOTE_RELEASES_DIR, commit)
|
||
run(ssh_cmd(host, user, "mkdir -p %s" % REMOTE_RELEASES_DIR), check=False)
|
||
transfer(host, user, tarball, remote_tar, sha)
|
||
transfer(host, user, HOST_SCRIPT, remote_script, sha256_of(HOST_SCRIPT))
|
||
print("执行主机部署脚本（唯一脚本，staging 换目录 + 失败回滚）…")
|
||
r = subprocess.run(
|
||
ssh_cmd(host, user,
|
||
"sh %s %s %s %s" % (remote_script, remote_tar, sha, commit)),
|
||
capture_output=True, text=True)
|
||
print(r.stdout)
|
||
if r.stderr:
|
||
print(r.stderr, file=sys.stderr)
|
||
if r.returncode != 0 or not parse_release_ok(r.stdout, commit):
|
||
print("主机部署未返回有效的 RELEASE_OK，部署失败（主机侧已回滚或未动）。",
|
||
file=sys.stderr)
|
||
sys.exit(1)
|
||
# 独立二次核对：不只信主机脚本的打印，再 ssh 读一次 live 清单。
|
||
live = subprocess.run(
|
||
ssh_cmd(host, user, "cat /opt/codex-app-server/DEPLOYED_COMMIT"),
|
||
capture_output=True, text=True)
|
||
if live_commit(live.stdout) != commit:
|
||
print("二次核对失败：live DEPLOYED_COMMIT=%s，期望 %s。"
|
||
"请立即用 deploy/rollback.sh 检查，不要再推新版。"
|
||
% (live.stdout.strip(), commit), file=sys.stderr)
|
||
sys.exit(1)
|
||
print("部署成功且二次核对通过：commit=%s sha256=%s" % (commit, sha))
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|