Ops batch: deploy automation, backup restore drill, monitoring, multi-user verification
Deploy: deploy.sh builds tarball, backs up current code (keep 3), deploys, restarts, health-gates 60s (ok/backend=postgres/db_ok), auto-rolls back on failure; rollback.sh restores latest backup. Rollback is code-only, DB migrations are additive-only so old code can still read new columns. Backup: restore_postgres refuses to restore into the production DB name unless explicitly allowed (drill never allows it); restore-drill.sh backs up fresh, restores to a test DB, compares tasks/projects/users/audit row counts, drops the test DB. Monitoring: worker writes heartbeat file every loop (atomic replace, failure only logged); GET /health adds worker_last_seen_seconds_ago + worker_ok (180s default, env tunable, still HTTP 200 when worker is stale); monitor.sh checks server/worker/health/pg_isready/docker-postgres/disk/ app dir as one JSON line, exit non-zero on any fail, plus systemd timer every 5 minutes. Push alerting to chat still needs a separate channel decision (SSH key or webhook URL). Multi-user: verify-multiuser.py creates test users/projects in memory-key only, checks RBAC via real HTTP API (owner/ maintainer/contributor/viewer, cross-project isolation, merge 200/403/409, no-key 401), test tasks carry a never- satisfiable dependency so the production worker never claims them, cleanup deletes only multiuser-test- rows (audit log is append-only and keeps test entries by design). Tests: host 27 pass, cloud 298 pass (8 skipped are environment-dependent real-Postgres tests).
This commit is contained in:
host committed
1 parent
6afef78ca8
commit
89a918b543
14 files changed
+1386
-2
No files matched your search
Executable
+106
@@ -0,0 +1,106 @@
|
||||
#!/bin/sh
|
||||
# 备份恢复演练：主机侧运行（root，或能读 /etc/codex-app-server/env 的用户）。
|
||||
#
|
||||
# 用法（主机上）：sh /opt/codex-app-server/deploy/restore-drill.sh
|
||||
# 可选环境变量：BACKUP_DIR（默认 /var/lib/codex-app-server/backups）。
|
||||
#
|
||||
# 步骤：
|
||||
# 1. 跑一次 backup_postgres 生成新鲜备份（.sql）。
|
||||
# 2. 新建一个测试库 codex_control_restore_test_<时间戳>，用
|
||||
# cloud.backup.restore_postgres 把备份恢复进去。
|
||||
# 3. 比对生产库与测试库的 tasks / projects / users / audit_log 行数，
|
||||
# 行数相等即通过；只比行数，不比对具体内容，不打印任何行内容。
|
||||
# 4. 比对完立即删掉测试库（trap 保证即使中途失败也删）。
|
||||
#
|
||||
# 安全红线（写进脚本逻辑里，不是只写在注释里）：
|
||||
# - 恢复目标永远是新生成的测试库名，绝不等于生产库名；restore_postgres
|
||||
# 本身也会拒绝目标库名等于 CLOUD_DB_NAME（allow_production 绝不在
|
||||
# 本脚本里设为 True），双重保险。
|
||||
# - 本脚本绝不恢复到生产库，绝不删生产库；删的只会是本次生成的测试库。
|
||||
# - 全程只打印 PASS/FAIL 与行数，不打印密码，不打印备份内容。
|
||||
# - 本脚本以 root 身份读 env 文件是为了拿到数据库连接参数与密码，
|
||||
# 密码只经 PGPASSWORD 环境变量传给 psql/python 子进程，绝不出现在
|
||||
# 命令行参数里，也绝不被 echo 出来。
|
||||
set -eu
|
||||
|
||||
APP_DIR=${APP_DIR:-/opt/codex-app-server}
|
||||
BACKUP_DIR=${BACKUP_DIR:-/var/lib/codex-app-server/backups}
|
||||
ENV_FILE=${ENV_FILE:-/etc/codex-app-server/env}
|
||||
|
||||
if [ ! -f "$ENV_FILE" ]; then
|
||||
echo "FAIL：找不到 env 文件：$ENV_FILE" >&2
|
||||
exit 1
|
||||
fi
|
||||
# shellcheck disable=SC1090
|
||||
set -a
|
||||
. "$ENV_FILE"
|
||||
set +a
|
||||
if [ -z "${CLOUD_DB_NAME:-}" ] || [ -z "${CLOUD_DB_USER:-}" ]; then
|
||||
echo "FAIL：env 文件里缺少 CLOUD_DB_NAME 或 CLOUD_DB_USER" >&2
|
||||
exit 1
|
||||
fi
|
||||
export PGPASSWORD="${CLOUD_DB_PASSWORD:-}"
|
||||
HOST_ARG=${CLOUD_DB_HOST:-127.0.0.1}
|
||||
PORT_ARG=${CLOUD_DB_PORT:-5432}
|
||||
|
||||
STAMP=$(date +%Y%m%d%H%M%S)
|
||||
TEST_DB="codex_control_restore_test_$STAMP"
|
||||
|
||||
drop_test_db() {
|
||||
# trap 保证：无论演练成功失败，测试库都会被删掉，不留垃圾库。
|
||||
psql -h "$HOST_ARG" -p "$PORT_ARG" -U "$CLOUD_DB_USER" \
|
||||
-d postgres -c "DROP DATABASE IF EXISTS \"$TEST_DB\"" >/dev/null 2>&1 || true
|
||||
}
|
||||
trap drop_test_db EXIT
|
||||
|
||||
# 1. 新鲜备份（backup_postgres 的连接参数与密码都走环境变量，
|
||||
# 密码只经 PGPASSWORD 传给子进程，绝不出现在命令行或输出里）。
|
||||
mkdir -p "$BACKUP_DIR"
|
||||
BACKUP_FILE=$(python3 - "$APP_DIR" "$BACKUP_DIR" <<'PY'
|
||||
import sys
|
||||
sys.path.insert(0, sys.argv[1])
|
||||
from cloud.backup import backup_postgres
|
||||
print(backup_postgres(sys.argv[2]))
|
||||
PY
|
||||
)
|
||||
if [ ! -f "$BACKUP_FILE" ]; then
|
||||
echo "FAIL：备份生成失败" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "备份已生成（只打印路径，不打印内容）：$BACKUP_FILE"
|
||||
|
||||
# 2. 建测试库并恢复
|
||||
psql -h "$HOST_ARG" -p "$PORT_ARG" -U "$CLOUD_DB_USER" \
|
||||
-d postgres -c "CREATE DATABASE \"$TEST_DB\"" >/dev/null
|
||||
python3 - "$APP_DIR" "$BACKUP_FILE" "$TEST_DB" <<'PY'
|
||||
import sys
|
||||
sys.path.insert(0, sys.argv[1])
|
||||
from cloud.backup import restore_postgres
|
||||
# allow_production 故意不传（默认 False）：目标库名绝不能等于生产库名。
|
||||
print(restore_postgres(sys.argv[2], sys.argv[3]))
|
||||
PY
|
||||
echo "恢复完成，目标测试库：$TEST_DB"
|
||||
|
||||
# 3. 行数比对（只比行数，不打印行内容）
|
||||
FAIL=0
|
||||
for TABLE in tasks projects users audit_log; do
|
||||
PROD_N=$(psql -h "$HOST_ARG" -p "$PORT_ARG" -U "$CLOUD_DB_USER" \
|
||||
-d "$CLOUD_DB_NAME" -t -A -c "SELECT COUNT(*) FROM $TABLE" | tr -d ' \n')
|
||||
REST_N=$(psql -h "$HOST_ARG" -p "$PORT_ARG" -U "$CLOUD_DB_USER" \
|
||||
-d "$TEST_DB" -t -A -c "SELECT COUNT(*) FROM $TABLE" | tr -d ' \n')
|
||||
if [ "$PROD_N" = "$REST_N" ]; then
|
||||
echo "PASS $TABLE：生产库=$PROD_N 恢复库=$REST_N"
|
||||
else
|
||||
echo "FAIL $TABLE：生产库=$PROD_N 恢复库=$REST_N" >&2
|
||||
FAIL=1
|
||||
fi
|
||||
done
|
||||
|
||||
# 4. 删测试库由 trap 执行；这里先显式删一次，trap 再删是空操作。
|
||||
drop_test_db
|
||||
trap - EXIT
|
||||
if [ "$FAIL" = "1" ]; then
|
||||
echo "恢复演练 FAIL：有表行数不一致" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "恢复演练 PASS：全部 4 张表行数一致，测试库已删除"
|
||||
Reference in new issue
Block a user