* 部署自动化:唯一发布链路 deploy.py + host_deploy.sh,sha/commit 双重验真,失败回滚 旧链路三次栽过的坑:通用 tar 路径导致旧包误部署、传输不验 sha、 主机脚本多份漂移、health 通过不等于新代码上线。本单把四点变硬门禁: - 产物唯一命名 codex-app-server-<commit>.tar.gz + .sha256,远端唯一路径 - git archive 产物 + DEPLOYED_COMMIT 清单,默认拒绝脏工作区 - 传输链 scp -> scp_run -> chunk 分片,每路传完远端验 sha,不符删掉重来 - 主机侧唯一脚本:先验 sha/清单,staging 解包 + 编译检查,换目录上线, work/ 原样保留,health + 上线 commit 不符即回滚并退出非零 - deploy.py 成功后独立 ssh 再读一次 live 清单二次核对 - CI:host-verify 加 deploy/tests;release.yml 只在手动/tag + production environment 审批时部署,主干推送绝不自动部署生产 - deploy.sh 标记为旧链路弃用 - health 的 commit 字段已在父提交 HA 阶段 0 里随 server.py 落地, 本分支整体包含它;commit 为空时只靠文件核对,门禁同样成立 测试:host 32、cloud 487(4 跳过)、deploy 9,全部通过; host_deploy.sh 仿真覆盖成功/坏 sha/清单不符/health commit 不符回滚。 生产实机部署未做,等确认后再用新链路推。 * HA stage 1: worktree_host + queued cross-host PR jobs --------- Co-authored-by: host <host@test.local>
148 lines
6.5 KiB
Python
148 lines
6.5 KiB
Python
"""host_deploy.sh 的仿真测试:假 systemctl / 假 curl,不碰真实主机。
|
||
|
||
每个用例都在临时目录里建一个假 APP_DIR,host_deploy.sh 的
|
||
CODEX_DEPLOY_* 环境变量把它指过去。生产默认值不受影响。
|
||
"""
|
||
import hashlib
|
||
import json
|
||
import os
|
||
import subprocess
|
||
import tarfile
|
||
import tempfile
|
||
import unittest
|
||
|
||
REPO_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(
|
||
os.path.abspath(__file__))))
|
||
HOST_SCRIPT = os.path.join(REPO_ROOT, "deploy", "host_deploy.sh")
|
||
|
||
|
||
def make_tarball(source_dir, tarball):
|
||
with tarfile.open(tarball, "w:gz", format=tarfile.PAX_FORMAT) as tf:
|
||
for root, dirs, files in os.walk(source_dir):
|
||
dirs.sort()
|
||
for name in sorted(files):
|
||
full = os.path.join(root, name)
|
||
tf.add(full, arcname=os.path.relpath(full, source_dir))
|
||
with open(tarball, "rb") as f:
|
||
return hashlib.sha256(f.read()).hexdigest()
|
||
|
||
|
||
class HostDeploySim(unittest.TestCase):
|
||
def setUp(self):
|
||
self.tmp = tempfile.TemporaryDirectory()
|
||
base = self.tmp.name
|
||
self.app = os.path.join(base, "codex-app-server")
|
||
os.makedirs(os.path.join(self.app, "work"))
|
||
with open(os.path.join(self.app, "DEPLOYED_COMMIT"), "w") as f:
|
||
f.write("oldcommit\n")
|
||
with open(os.path.join(self.app, "work", "keep.txt"), "w") as f:
|
||
f.write("keep\n")
|
||
with open(os.path.join(self.app, "old.py"), "w") as f:
|
||
f.write("x = 0\n")
|
||
# 假 systemctl:is-active 一律失败(走 cloud-worker 分支),其余成功。
|
||
fake_bin = os.path.join(base, "fakebin")
|
||
os.makedirs(fake_bin)
|
||
with open(os.path.join(fake_bin, "systemctl"), "w") as f:
|
||
f.write("#!/bin/sh\n"
|
||
"echo \"$@\" >> \"$FAKE_SYSTEMCTL_LOG\"\n"
|
||
"if [ \"$1\" = \"is-active\" ]; then exit 1; fi\n"
|
||
"exit 0\n")
|
||
with open(os.path.join(fake_bin, "curl"), "w") as f:
|
||
f.write("#!/bin/sh\ncat \"$FAKE_HEALTH_JSON\"\n")
|
||
os.chmod(os.path.join(fake_bin, "systemctl"), 0o755)
|
||
os.chmod(os.path.join(fake_bin, "curl"), 0o755)
|
||
self.fake_bin = fake_bin
|
||
self.health_path = os.path.join(base, "health.json")
|
||
self.log_path = os.path.join(base, "systemctl.log")
|
||
open(self.log_path, "w").close()
|
||
self.env = dict(os.environ)
|
||
self.env["PATH"] = fake_bin + os.pathsep + os.environ["PATH"]
|
||
self.env["CODEX_DEPLOY_APP_DIR"] = self.app
|
||
self.env["CODEX_DEPLOY_BACKUP_DIR"] = os.path.join(base, "backups")
|
||
self.env["CODEX_DEPLOY_STAGING_DIR"] = os.path.join(base, "staging")
|
||
self.env["CODEX_DEPLOY_OLD_DIR"] = os.path.join(base, "old")
|
||
self.env["CODEX_DEPLOY_FAILED_DIR"] = os.path.join(base, "failed")
|
||
self.env["CODEX_DEPLOY_PRESERVE_DIR"] = os.path.join(base, "preserve")
|
||
self.env["CODEX_DEPLOY_HEALTH_INTERVAL"] = "0.1"
|
||
self.env["CODEX_DEPLOY_HEALTH_ATTEMPTS"] = "3"
|
||
self.env["FAKE_HEALTH_JSON"] = self.health_path
|
||
self.env["FAKE_SYSTEMCTL_LOG"] = self.log_path
|
||
|
||
def tearDown(self):
|
||
self.tmp.cleanup()
|
||
|
||
def set_health(self, payload):
|
||
with open(self.health_path, "w") as f:
|
||
json.dump(payload, f)
|
||
|
||
def build_new_tarball(self, commit="newcommit"):
|
||
src = os.path.join(self.tmp.name, "src")
|
||
os.makedirs(src, exist_ok=True)
|
||
with open(os.path.join(src, "DEPLOYED_COMMIT"), "w") as f:
|
||
f.write(commit + "\n")
|
||
with open(os.path.join(src, "new.py"), "w") as f:
|
||
f.write("x = 1\n")
|
||
tarball = os.path.join(self.tmp.name, "release.tar.gz")
|
||
sha = make_tarball(src, tarball)
|
||
return tarball, sha
|
||
|
||
def run_host(self, tarball, sha, commit):
|
||
return subprocess.run(
|
||
["sh", HOST_SCRIPT, tarball, sha, commit],
|
||
capture_output=True, text=True, env=self.env)
|
||
|
||
def live_commit(self):
|
||
with open(os.path.join(self.app, "DEPLOYED_COMMIT")) as f:
|
||
return f.read().strip()
|
||
|
||
def test_success_swaps_and_preserves_work(self):
|
||
self.set_health({"ok": True, "backend": "postgres", "db_ok": True,
|
||
"commit": "newcommit"})
|
||
tarball, sha = self.build_new_tarball()
|
||
r = self.run_host(tarball, sha, "newcommit")
|
||
self.assertEqual(r.returncode, 0, r.stdout + r.stderr)
|
||
self.assertIn("RELEASE_OK", r.stdout)
|
||
self.assertIn("commit=newcommit", r.stdout)
|
||
self.assertEqual(self.live_commit(), "newcommit")
|
||
with open(os.path.join(self.app, "work", "keep.txt")) as f:
|
||
self.assertEqual(f.read().strip(), "keep")
|
||
self.assertTrue(os.path.exists(os.path.join(self.app, "new.py")))
|
||
self.assertFalse(os.path.exists(os.path.join(self.app, "old.py")))
|
||
backups = os.listdir(self.env["CODEX_DEPLOY_BACKUP_DIR"])
|
||
self.assertEqual(len(backups), 1)
|
||
self.assertIn("oldcommit", backups[0])
|
||
|
||
def test_bad_sha_never_touches_live(self):
|
||
self.set_health({"ok": True, "backend": "postgres", "db_ok": True,
|
||
"commit": "newcommit"})
|
||
tarball, _sha = self.build_new_tarball()
|
||
r = self.run_host(tarball, "0" * 64, "newcommit")
|
||
self.assertNotEqual(r.returncode, 0)
|
||
self.assertEqual(self.live_commit(), "oldcommit")
|
||
self.assertFalse(os.path.exists(self.env["CODEX_DEPLOY_BACKUP_DIR"])
|
||
and os.listdir(self.env["CODEX_DEPLOY_BACKUP_DIR"]))
|
||
|
||
def test_manifest_mismatch_never_touches_live(self):
|
||
self.set_health({"ok": True, "backend": "postgres", "db_ok": True,
|
||
"commit": "newcommit"})
|
||
tarball, sha = self.build_new_tarball(commit="othercommit")
|
||
r = self.run_host(tarball, sha, "newcommit")
|
||
self.assertNotEqual(r.returncode, 0)
|
||
self.assertEqual(self.live_commit(), "oldcommit")
|
||
|
||
def test_health_failure_rolls_back(self):
|
||
# health 通过但 commit 是旧的:这是旧链路会误报成功的情况,
|
||
# 新链路必须回滚,live 回到 oldcommit。
|
||
self.set_health({"ok": True, "backend": "postgres", "db_ok": True,
|
||
"commit": "oldcommit"})
|
||
tarball, sha = self.build_new_tarball()
|
||
r = self.run_host(tarball, sha, "newcommit")
|
||
self.assertNotEqual(r.returncode, 0)
|
||
self.assertEqual(self.live_commit(), "oldcommit")
|
||
with open(os.path.join(self.app, "work", "keep.txt")) as f:
|
||
self.assertEqual(f.read().strip(), "keep")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
unittest.main()
|