deploy: add git-pull source for host deployment
Host can now fetch code directly from GitHub instead of receiving a VM-built tarball, bypassing the VM->host scp/kex failures and stale-tar risk. Design: VM sends only two small scripts (host_git_fetch.sh + host_deploy.sh, sha-verified via the existing transfer chain, now with a chunk_key fallback that does not need SSH_PASSWORD); host clones via a read-only deploy key (SSH only, HTTPS refused, key must exist with mode 0600), checks out the exact 40-char commit, verifies HEAD, builds a tar in the same format as the VM release (DEPLOYED_COMMIT + RELEASE.json, member names without ./ prefix so host_deploy.sh tar -O works), then calls the unchanged host_deploy.sh for staging swap, backup, health and rollback. Git fetch tolerates the known intermittent host->GitHub degradation: 120s timeout, 3 attempts, 10s sleep, live untouched until host_deploy runs. VM gates: clean tree, tests, commit must be on origin/main, SSH repo only, full sha only; --source tar remains the default. Tests: deploy suite OK, including 6 new host_git_fetch simulations (success with clean tar members, unknown commit, key permissions/missing, HTTPS refusal, short sha) and 4 new deploy.py git-mode helpers plus chunk_key fallback; git-mode dry-run verified against HEAD 630d14f.
This commit is contained in:
host committed
1 parent
630d14fe07
commit
e8e256f6e5
5 files changed
+542
-7
No files matched your search
+140
-7
@@ -26,6 +26,14 @@
|
||||
python3 deploy/deploy.py <host> --build-only [--out-dir DIR]
|
||||
python3 deploy/deploy.py <host> --tarball <path> --commit <sha>
|
||||
python3 deploy/deploy.py <host> --skip-tests # CI 里测试已跑过时用
|
||||
python3 deploy/deploy.py <host> --source git [--repo <ssh-url>]
|
||||
# git 源：VM 不建 tar、不传大包，只传两个小脚本（host_git_fetch.sh
|
||||
# + host_deploy.sh）并让主机用只读 deploy key 直拉 GitHub；
|
||||
# 主机在 staging 里 checkout 精确 commit、验 HEAD、本地建包，
|
||||
# 再调 host_deploy.sh 换目录（tar 路径仍是默认，不改默认前
|
||||
# 必须显式 --source git）。主机凭证不在本脚本里：deploy key
|
||||
# 私钥只放主机（默认 /root/.ssh/id_codex_github，0600），
|
||||
# 公钥以只读方式加到 GitHub 仓库 Settings > Deploy keys。
|
||||
|
||||
密钥规则：本脚本绝不接受密码参数、不读任何 env 文件、不打印任何密钥。
|
||||
SSH 密码只由调用方已有的通道处理（key，或 ~/workspace/ssh-helpers 的
|
||||
@@ -45,6 +53,9 @@ from datetime import datetime, timezone
|
||||
|
||||
REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
HOST_SCRIPT = os.path.join(REPO_ROOT, "deploy", "host_deploy.sh")
|
||||
HOST_GIT_FETCH_SCRIPT = os.path.join(REPO_ROOT, "deploy", "host_git_fetch.sh")
|
||||
DEFAULT_GIT_REPO = "git@github.com:openqj/codex-app-server.git"
|
||||
DEFAULT_GIT_KEY = "/root/.ssh/id_codex_github"
|
||||
HELPER_DIR = os.path.expanduser("~/workspace/ssh-helpers")
|
||||
REMOTE_RELEASES_DIR = "/tmp/codex-app-server-releases"
|
||||
|
||||
@@ -63,6 +74,28 @@ def git_commit(ref):
|
||||
return run(["git", "rev-parse", ref], cwd=REPO_ROOT).stdout.strip()
|
||||
|
||||
|
||||
def is_full_sha(value):
|
||||
"""git 源只接受 40 位完整 sha：短 sha 有歧义，主机 checkout
|
||||
前后都必须能用它唯一确定一个提交。"""
|
||||
return bool(re.fullmatch(r"[0-9a-f]{40}", value or ""))
|
||||
|
||||
|
||||
def is_ssh_git_url(url):
|
||||
"""git 源只接受 SSH 形式：HTTPS 需要 token，不在本模式内；
|
||||
仓库是私有的，裸 https 拉不到，token 绝不能拼进 URL。"""
|
||||
return bool(url) and (
|
||||
url.startswith("git@github.com:") or
|
||||
url.startswith("ssh://git@github.com/"))
|
||||
|
||||
|
||||
def commit_on_origin_main(commit):
|
||||
"""commit 是否已在 origin/main 里：主机只能拉到已推的提交，
|
||||
本地有但未推的 HEAD 用 git 源部署必失败，先在 VM 侧拦住。"""
|
||||
r = run(["git", "merge-base", "--is-ancestor", commit, "origin/main"],
|
||||
cwd=REPO_ROOT, check=False)
|
||||
return r.returncode == 0
|
||||
|
||||
|
||||
def check_clean_tree(allow_dirty):
|
||||
"""已提交内容与工作区不一致时，默认拒绝；--allow-dirty 只告警，
|
||||
且部署的仍是 HEAD 的 archive，脏文件不会被偷偷带上去。"""
|
||||
@@ -247,23 +280,32 @@ def remote_rm(host, user, remote_path):
|
||||
|
||||
|
||||
def transfer(host, user, local, remote, expected_sha):
|
||||
"""传输链：scp -> helper scp -> chunk 分片。每路传完都验远端 sha，
|
||||
不符删掉重来；全部失败才退出，绝不把错包交给主机脚本。"""
|
||||
"""传输链：scp -> chunk_key 分片（密钥版）-> chunk 分片（密码版）。
|
||||
每路传完都验远端 sha，不符删掉重来；全部失败才退出，
|
||||
绝不把错包交给主机脚本。chunk_key 一路不依赖 SSH_PASSWORD：
|
||||
它用部署密钥走显式 ProxyCommand，是 VM 侧实测最稳的小文件
|
||||
传输路（git 源只传两个小脚本时尤其重要：大 tar 的 scp 在
|
||||
kex 阶段被 egress 代理 reset 的概率远高于小脚本，但小脚本
|
||||
也不能只押 scp 一路）。"""
|
||||
attempts = []
|
||||
# 1. scp（含 helper 路由，由 scp_cmd 内部决定）
|
||||
attempts.append(("scp", scp_cmd(host, user, local, remote)))
|
||||
chunk_key = helper_path("chunk_send_key_explicit.py")
|
||||
if chunk_key:
|
||||
attempts.append(("chunk_key", None)) # 特殊处理，见下
|
||||
chunk = helper_path("chunk_send.py")
|
||||
if chunk and os.environ.get("SSH_PASSWORD"):
|
||||
remote_b64 = remote + ".b64"
|
||||
attempts.append(("chunk", None)) # 特殊处理，见下
|
||||
remote_b64 = remote + ".b64"
|
||||
for name, cmd in attempts:
|
||||
print("传输尝试：%s -> %s" % (name, remote))
|
||||
if name == "chunk":
|
||||
if name in ("chunk", "chunk_key"):
|
||||
helper = chunk_key if name == "chunk_key" else chunk
|
||||
r = subprocess.run(
|
||||
["python3", chunk, local, remote_b64, host, user],
|
||||
["python3", helper, local, remote_b64, host, user],
|
||||
capture_output=True, text=True)
|
||||
if r.returncode != 0:
|
||||
print("chunk 传输失败，换下一路。", file=sys.stderr)
|
||||
print("%s 传输失败，换下一路。" % name, file=sys.stderr)
|
||||
continue
|
||||
r = subprocess.run(
|
||||
ssh_cmd(host, user,
|
||||
@@ -271,7 +313,7 @@ def transfer(host, user, local, remote, expected_sha):
|
||||
% (remote_b64, remote, remote)),
|
||||
capture_output=True, text=True)
|
||||
if r.returncode != 0:
|
||||
print("chunk 解码失败，换下一路。", file=sys.stderr)
|
||||
print("%s 解码失败，换下一路。" % name, file=sys.stderr)
|
||||
continue
|
||||
else:
|
||||
r = subprocess.run(cmd, capture_output=True, text=True)
|
||||
@@ -306,6 +348,87 @@ def run_tests():
|
||||
cwd=REPO_ROOT)
|
||||
|
||||
|
||||
def deploy_via_git(args):
|
||||
"""git 源部署：VM 只传两个小脚本，主机直拉 GitHub。
|
||||
|
||||
门禁顺序与 tar 路径同款强度，只是取包方换了主人：
|
||||
脏工作区、测试、commit 未推到 origin/main、repo 非 SSH、
|
||||
commit 非完整 sha，任一不符都拒绝，主机绝不动。
|
||||
主机侧 host_git_fetch.sh 负责重试、checkout 校验、本地建包，
|
||||
再调 host_deploy.sh 换目录；换目录后的 RELEASE_OK 解析与
|
||||
live 二次核对与 tar 路径完全一致。"""
|
||||
if args.tarball:
|
||||
print("--tarball 与 --source git 冲突：git 源不传包，请去掉 --tarball。",
|
||||
file=sys.stderr)
|
||||
sys.exit(2)
|
||||
if args.build_only:
|
||||
print("--source git 不在 VM 建包，--build-only 不适用。",
|
||||
file=sys.stderr)
|
||||
sys.exit(2)
|
||||
if not args.host and not args.dry_run:
|
||||
print("git 源需要 host，或用 --dry-run。", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
if not is_ssh_git_url(args.repo):
|
||||
print("git 源只接受 SSH 仓库（deploy key），HTTPS 需要 token：%s"
|
||||
% args.repo, file=sys.stderr)
|
||||
sys.exit(2)
|
||||
check_clean_tree(args.allow_dirty)
|
||||
if not args.skip_tests:
|
||||
run_tests()
|
||||
commit = git_commit(args.ref)
|
||||
if not is_full_sha(commit):
|
||||
print("git 源需要 40 位完整 commit，解析结果不符：%s" % commit,
|
||||
file=sys.stderr)
|
||||
sys.exit(2)
|
||||
if not commit_on_origin_main(commit):
|
||||
print("commit 未在 origin/main 里，主机拉不到：先推分支/合 PR 再推。",
|
||||
file=sys.stderr)
|
||||
print(" commit=%s" % commit, file=sys.stderr)
|
||||
sys.exit(2)
|
||||
print("git 源：repo=%s commit=%s git_key=%s（只传路径）"
|
||||
% (args.repo, commit, args.git_key))
|
||||
if args.dry_run:
|
||||
print("[dry-run] 将传 host_git_fetch.sh 与 host_deploy.sh 到 "
|
||||
"%s@%s:%s（唯一路径，传完验 sha）" % (args.user, args.host,
|
||||
REMOTE_RELEASES_DIR))
|
||||
print("[dry-run] 将执行：sh host_git_fetch.sh <repo> <commit> "
|
||||
"<git_key> <host_deploy.sh>，主机直拉、校验、本地建包、"
|
||||
"换目录，成功须打印 RELEASE_OK 且 commit 一致，否则回滚。")
|
||||
return 0
|
||||
host, user = args.host, args.user
|
||||
remote_git_fetch = "%s/host_git_fetch-%s.sh" % (REMOTE_RELEASES_DIR, commit)
|
||||
remote_host_deploy = "%s/host_deploy-%s.sh" % (REMOTE_RELEASES_DIR, commit)
|
||||
run(ssh_cmd(host, user, "mkdir -p %s" % REMOTE_RELEASES_DIR), check=False)
|
||||
transfer(host, user, HOST_GIT_FETCH_SCRIPT, remote_git_fetch,
|
||||
sha256_of(HOST_GIT_FETCH_SCRIPT))
|
||||
transfer(host, user, HOST_SCRIPT, remote_host_deploy,
|
||||
sha256_of(HOST_SCRIPT))
|
||||
print("执行主机 git 取包 + 部署脚本（staging 取包 + 换目录 + 失败回滚）…")
|
||||
r = subprocess.run(
|
||||
ssh_cmd(host, user,
|
||||
"sh %s %s %s %s %s" % (remote_git_fetch, args.repo, commit,
|
||||
args.git_key, remote_host_deploy)),
|
||||
capture_output=True, text=True)
|
||||
print(r.stdout)
|
||||
if r.stderr:
|
||||
print(r.stderr, file=sys.stderr)
|
||||
if r.returncode != 0 or not parse_release_ok(r.stdout, commit):
|
||||
print("主机 git 部署未返回有效的 RELEASE_OK，部署失败（主机侧已回滚或未动）。",
|
||||
file=sys.stderr)
|
||||
sys.exit(1)
|
||||
live = subprocess.run(
|
||||
ssh_cmd(host, user, "cat /opt/codex-app-server/DEPLOYED_COMMIT"),
|
||||
capture_output=True, text=True)
|
||||
if live_commit(live.stdout) != commit:
|
||||
print("二次核对失败：live DEPLOYED_COMMIT=%s，期望 %s。"
|
||||
"请立即用 deploy/rollback.sh 检查，不要再推新版。"
|
||||
% (live.stdout.strip(), commit), file=sys.stderr)
|
||||
sys.exit(1)
|
||||
print("部署成功且二次核对通过：source=git commit=%s repo=%s"
|
||||
% (commit, args.repo))
|
||||
return 0
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
parser = argparse.ArgumentParser(description="codex-app-server 统一发布入口")
|
||||
parser.add_argument("host", nargs="?", help="部署目标主机")
|
||||
@@ -313,6 +436,13 @@ def main(argv=None):
|
||||
parser.add_argument("--ref", default="HEAD")
|
||||
parser.add_argument("--tarball", help="直接部署已有发布包（CI 产物）")
|
||||
parser.add_argument("--commit", help="配合 --tarball 指定期望 commit")
|
||||
parser.add_argument("--source", choices=("tar", "git"), default="tar",
|
||||
help="取包方式：tar（默认，VM 建包传包）或 git"
|
||||
"（主机用只读 deploy key 直拉 GitHub，见本文件头注释）")
|
||||
parser.add_argument("--repo", default=DEFAULT_GIT_REPO,
|
||||
help="git 源仓库（只接受 SSH 形式，默认 openqj/codex-app-server）")
|
||||
parser.add_argument("--git-key", default=DEFAULT_GIT_KEY,
|
||||
help="git 源主机 deploy key 私钥路径（只传路径，不传内容）")
|
||||
parser.add_argument("--out-dir", default=os.path.join(
|
||||
REPO_ROOT, "work", "external-install", "releases"))
|
||||
parser.add_argument("--allow-dirty", action="store_true")
|
||||
@@ -321,6 +451,9 @@ def main(argv=None):
|
||||
parser.add_argument("--dry-run", action="store_true")
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
if args.source == "git":
|
||||
return deploy_via_git(args)
|
||||
|
||||
if args.tarball:
|
||||
commit = args.commit or tarball_commit(args.tarball)
|
||||
tarball, sha = args.tarball, sha256_of(args.tarball)
|
||||
|
||||
Reference in new issue
Block a user