HA stage 0: health reads workers registry, daily pg_dump timer, backup freshness in monitor (#48)

Co-authored-by: host <host@test.local>
This commit is contained in:
openqjandhost authored and GitHub committed 2026-10-26 23:51:54 +08:00
1 parent 275de39148
commit 455513dd33
6 files changed
+150 -10

No files matched your search

+23 -4
View File
@@ -18,6 +18,9 @@
# docker_postgres:docker ps 里 codex-postgres 在运行
# disk:磁盘使用率 < 90%(默认看 /var/lib/codex-app-server,不存在则看 /)
# app_dir:/opt/codex-app-server 目录存在
# backup:最近的 cloud-backup-*.sql 在 36 小时内(每天一次定时备份,
# HA 阶段 0;目录默认 /var/lib/codex-app-server/backups,可用
# MONITOR_BACKUP_DIR 覆盖)
#
# 输出示例(单行 JSON,值只会是 pass/fail,不含任何密钥):
# {"overall": "pass", "checks": {"server": "pass", ...}}
@@ -105,16 +108,32 @@ check_app_dir() {
if [ -d "$APP_DIR" ]; then echo pass; else echo fail; fi
}
check_backup() {
# 备份新鲜度:BACKUP_DIR 里最近的 cloud-backup-*.sql 在 36 小时内
# (每天一次,36 小时容忍一次失败 + 时区/夏令时抖动)。目录或文件
# 缺失一律 fail——备份制度断了必须在 monitor 里可见,不能静默。
BACKUP_DIR=${MONITOR_BACKUP_DIR:-/var/lib/codex-app-server/backups}
NEWEST=$(ls -t "$BACKUP_DIR"/cloud-backup-*.sql 2>/dev/null | head -1 || true)
if [ -z "$NEWEST" ]; then echo fail; return; fi
MTIME=$(stat -c %Y "$NEWEST" 2>/dev/null || true)
case "$MTIME" in
''|*[!0-9]*) echo fail; return ;;
esac
NOW=$(date +%s)
AGE=$((NOW - MTIME))
if [ "$AGE" -lt 129600 ]; then echo pass; else echo fail; fi
}
run_checks() {
S=$(check_server); W=$(check_worker); H=$(check_health)
P=$(check_postgres); D=$(check_docker_postgres)
K=$(check_disk); A=$(check_app_dir)
K=$(check_disk); A=$(check_app_dir); B=$(check_backup)
OVERALL=pass
for v in "$S" "$W" "$H" "$P" "$D" "$K" "$A"; do
for v in "$S" "$W" "$H" "$P" "$D" "$K" "$A" "$B"; do
if [ "$v" != "pass" ]; then OVERALL=fail; break; fi
done
printf '{"overall": "%s", "checks": {"server": "%s", "worker": "%s", "health": "%s", "postgres": "%s", "docker_postgres": "%s", "disk": "%s", "app_dir": "%s"}}\n' \
"$OVERALL" "$S" "$W" "$H" "$P" "$D" "$K" "$A"
printf '{"overall": "%s", "checks": {"server": "%s", "worker": "%s", "health": "%s", "postgres": "%s", "docker_postgres": "%s", "disk": "%s", "app_dir": "%s", "backup": "%s"}}\n' \
"$OVERALL" "$S" "$W" "$H" "$P" "$D" "$K" "$A" "$B"
if [ "$OVERALL" = "pass" ]; then return 0; else return 1; fi
}