diff --git a/cloud/server.py b/cloud/server.py index 49ed531..29973d6 100644 --- a/cloud/server.py +++ b/cloud/server.py @@ -205,10 +205,33 @@ def _db_backend_name(): return (os.environ.get("CLOUD_DB_BACKEND") or "sqlite").strip().lower() +def _deployed_commit(): + """当前上线代码的 commit：deploy.py 部署时在包根写的 DEPLOYED_COMMIT。 + + 文件不存在、读失败、内容为空时返回 None（老包、本地开发都走这支）， + health 绝不因此失败。commit 是公开的版本标识，不是密钥。 + 路径取 cloud/ 的父目录（即仓库根），不依赖当前工作目录。""" + path = os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + "DEPLOYED_COMMIT") + try: + with open(path, encoding="utf-8") as f: + value = f.read().strip() + except Exception: + return None + return value or None + + def _worker_heartbeat_status(server=None): """读 worker 运维心跳，返回 (age_seconds_or_None, worker_ok)。 - 心跳文件路径：测试注入优先（server.worker_heartbeat_file）， + 优先读 workers 注册表（多机前提）：所有 worker 每轮都往 + workers 表写 last_seen（见 worker._registry_heartbeat），而文件 + 心跳只在本机存在——多 server 下 server 读不到别机 worker 的 + 文件，注册表是唯一跨机可靠的心跳源。表不存在、无行、查库 + 失败时回落文件心跳（单机旧行为与单测保持兼容）。 + + 文件心跳路径：测试注入优先（server.worker_heartbeat_file）， 其次环境变量 CLOUD_WORKER_HEARTBEAT_FILE，未设则默认 ./worker-heartbeat.json——与 worker 的默认路径保持一致，主机 部署时两边都要指到同一个主机路径（见 deploy/README）。 @@ -217,25 +240,40 @@ def _worker_heartbeat_status(server=None): 文件不存在、读失败、JSON 非法、last_seen 非数字时一律返回 (None, False)，不抛异常、不影响 health 本身的 200。 """ - path = None max_age = 180 if server is not None: - path = getattr(server, "worker_heartbeat_file", None) override = getattr(server, "worker_heartbeat_max_age_seconds", None) if override is not None: try: max_age = float(override) except (TypeError, ValueError): max_age = 180 - if not path: - path = os.environ.get("CLOUD_WORKER_HEARTBEAT_FILE") or \ - "./worker-heartbeat.json" if max_age == 180: try: max_age = float(os.environ.get( "CLOUD_WORKER_HEARTBEAT_MAX_AGE_SECONDS") or 180) except (TypeError, ValueError): max_age = 180 + db_path = getattr(server, "db_path", None) if server is not None else None + if db_path: + try: + conn = _connect(db_path) + try: + row = conn.execute( + "SELECT MAX(last_seen) FROM workers").fetchone() + finally: + conn.close() + if row is not None and row[0] is not None: + age = time.time() - float(row[0]) + return round(age, 1), bool(age <= max_age) + except Exception: + pass + path = None + if server is not None: + path = getattr(server, "worker_heartbeat_file", None) + if not path: + path = os.environ.get("CLOUD_WORKER_HEARTBEAT_FILE") or \ + "./worker-heartbeat.json" try: with open(path, encoding="utf-8") as f: payload = json.load(f) @@ -574,6 +612,9 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler): "backend": _db_backend_name(), "db_ok": db_ok, "worker_last_seen_seconds_ago": age, "worker_ok": worker_ok, + # 上线证据：health 通过不再单独证明新代码， + # 部署验真必须同时核对 commit 等于期望值（见 deploy.py）。 + "commit": _deployed_commit(), }) return if path.startswith("/oauth/"): diff --git a/cloud/tests/test_ops_batch.py b/cloud/tests/test_ops_batch.py index aa78c63..fdb985b 100644 --- a/cloud/tests/test_ops_batch.py +++ b/cloud/tests/test_ops_batch.py @@ -21,11 +21,13 @@ try: from cloud.backup import restore_postgres from cloud.server import create_server from cloud.worker import Worker + from cloud.workers import heartbeat_worker, register_worker except ImportError: sys.path.insert(0, str(Path(__file__).resolve().parents[2])) from cloud.backup import restore_postgres from cloud.server import create_server from cloud.worker import Worker + from cloud.workers import heartbeat_worker, register_worker def _fake_result(returncode=0, stdout="", stderr=""): @@ -187,6 +189,35 @@ class HealthWorkerFieldTestCase(unittest.TestCase): self.assertIsNone(body["worker_last_seen_seconds_ago"]) self.assertFalse(body["worker_ok"]) + def test_health_worker_db_registry_fresh_ok_without_file(self): + # 多机前提：文件心跳缺失时，DB 注册表的新鲜心跳同样算 ok。 + register_worker(self.db_path, "w-db", executor="docker", capacity=1) + heartbeat_worker(self.db_path, "w-db", executor="docker") + self.server.worker_heartbeat_file = os.path.join( + self.tmp.name, "no-such-hb.json") + status, body = self._health() + self.assertEqual(status, 200) + self.assertTrue(body["worker_ok"]) + self.assertIsNotNone(body["worker_last_seen_seconds_ago"]) + self.assertLess(body["worker_last_seen_seconds_ago"], 5) + + def test_health_worker_db_registry_stale_not_ok(self): + register_worker(self.db_path, "w-db", executor="docker", capacity=1) + import sqlite3 + conn = sqlite3.connect(self.db_path) + try: + conn.execute("UPDATE workers SET last_seen = ?", + (time.time() - 1000,)) + conn.commit() + finally: + conn.close() + self.server.worker_heartbeat_file = os.path.join( + self.tmp.name, "no-such-hb.json") + status, body = self._health() + self.assertEqual(status, 200) + self.assertFalse(body["worker_ok"]) + self.assertGreater(body["worker_last_seen_seconds_ago"], 500) + class MonitorSelfTestCase(unittest.TestCase): def test_monitor_self_test_passes(self): diff --git a/deploy/codex-pg-backup.service b/deploy/codex-pg-backup.service new file mode 100644 index 0000000..0d555c9 --- /dev/null +++ b/deploy/codex-pg-backup.service @@ -0,0 +1,17 @@ +# 每日 Postgres 备份服务（HA 阶段 0）。 +# +# 注意：unit 文件不在 deploy tar 里，改动后须单独 scp 到 +# /etc/systemd/system/ 再 daemon-reload + enable + start，不要只重启服务。 +# 本 service 以 root 跑，只因为要读 0600 的 /etc/codex-app-server/env； +# 备份文件由 UMask=0077 压成 600，目录由 pg-backup.sh 压成 700。 +[Unit] +Description=Daily Postgres backup for codex-app-server +# 注意：codex-postgres 是 docker 容器、不是 systemd 服务，这里不写 +# After/Wants；Postgres 不可达时 pg_dump 直接失败、service 失败可见。 + +[Service] +Type=oneshot +User=root +UMask=0077 +EnvironmentFile=/etc/codex-app-server/env +ExecStart=/bin/sh /opt/codex-app-server/deploy/pg-backup.sh diff --git a/deploy/codex-pg-backup.timer b/deploy/codex-pg-backup.timer new file mode 100644 index 0000000..8f82426 --- /dev/null +++ b/deploy/codex-pg-backup.timer @@ -0,0 +1,15 @@ +# 每日 Postgres 备份定时器（HA 阶段 0）。 +# +# 注意：unit 文件不在 deploy tar 里，改动后须单独 scp 到 +# /etc/systemd/system/ 再 daemon-reload + enable，不要只重启服务。 +# 19:00 UTC = 次日 03:00 CST（主机时区为 UTC 时以 UTC 为准，见注释）。 +[Unit] +Description=Daily Postgres backup timer for codex-app-server + +[Timer] +OnCalendar=*-*-* 19:00:00 UTC +Persistent=true +Unit=codex-pg-backup.service + +[Install] +WantedBy=timers.target diff --git a/deploy/monitor.sh b/deploy/monitor.sh index 595406f..d7d2de7 100755 --- a/deploy/monitor.sh +++ b/deploy/monitor.sh @@ -18,6 +18,9 @@ # docker_postgres：docker ps 里 codex-postgres 在运行 # disk：磁盘使用率 < 90%（默认看 /var/lib/codex-app-server，不存在则看 /） # app_dir：/opt/codex-app-server 目录存在 +# backup：最近的 cloud-backup-*.sql 在 36 小时内（每天一次定时备份， +# HA 阶段 0；目录默认 /var/lib/codex-app-server/backups，可用 +# MONITOR_BACKUP_DIR 覆盖） # # 输出示例（单行 JSON，值只会是 pass/fail，不含任何密钥）： # {"overall": "pass", "checks": {"server": "pass", ...}} @@ -105,16 +108,32 @@ check_app_dir() { if [ -d "$APP_DIR" ]; then echo pass; else echo fail; fi } +check_backup() { + # 备份新鲜度：BACKUP_DIR 里最近的 cloud-backup-*.sql 在 36 小时内 + # （每天一次，36 小时容忍一次失败 + 时区/夏令时抖动）。目录或文件 + # 缺失一律 fail——备份制度断了必须在 monitor 里可见，不能静默。 + BACKUP_DIR=${MONITOR_BACKUP_DIR:-/var/lib/codex-app-server/backups} + NEWEST=$(ls -t "$BACKUP_DIR"/cloud-backup-*.sql 2>/dev/null | head -1 || true) + if [ -z "$NEWEST" ]; then echo fail; return; fi + MTIME=$(stat -c %Y "$NEWEST" 2>/dev/null || true) + case "$MTIME" in + ''|*[!0-9]*) echo fail; return ;; + esac + NOW=$(date +%s) + AGE=$((NOW - MTIME)) + if [ "$AGE" -lt 129600 ]; then echo pass; else echo fail; fi +} + run_checks() { S=$(check_server); W=$(check_worker); H=$(check_health) P=$(check_postgres); D=$(check_docker_postgres) - K=$(check_disk); A=$(check_app_dir) + K=$(check_disk); A=$(check_app_dir); B=$(check_backup) OVERALL=pass - for v in "$S" "$W" "$H" "$P" "$D" "$K" "$A"; do + for v in "$S" "$W" "$H" "$P" "$D" "$K" "$A" "$B"; do if [ "$v" != "pass" ]; then OVERALL=fail; break; fi done - printf '{"overall": "%s", "checks": {"server": "%s", "worker": "%s", "health": "%s", "postgres": "%s", "docker_postgres": "%s", "disk": "%s", "app_dir": "%s"}}\n' \ - "$OVERALL" "$S" "$W" "$H" "$P" "$D" "$K" "$A" + printf '{"overall": "%s", "checks": {"server": "%s", "worker": "%s", "health": "%s", "postgres": "%s", "docker_postgres": "%s", "disk": "%s", "app_dir": "%s", "backup": "%s"}}\n' \ + "$OVERALL" "$S" "$W" "$H" "$P" "$D" "$K" "$A" "$B" if [ "$OVERALL" = "pass" ]; then return 0; else return 1; fi } diff --git a/deploy/pg-backup.sh b/deploy/pg-backup.sh new file mode 100644 index 0000000..d12b55a --- /dev/null +++ b/deploy/pg-backup.sh @@ -0,0 +1,17 @@ +#!/bin/sh +# 每日 Postgres 备份（由 codex-pg-backup.service 调，timer 每天触发）。 +# +# 连接参数只走 CLOUD_DB_* 环境变量（service 的 EnvironmentFile 指向 +# /etc/codex-app-server/env，0600 root）；密码绝不出现在命令行与日志里。 +# 备份目录默认 /var/lib/codex-app-server/backups，轮转只留最近 7 份 +# （每天一次 = 一周）；备份文件含生产库全量（含 GitHub App 配置）， +# 目录 700、service 的 UMask=0077 保证文件 600，绝不要放宽。 +set -eu + +APP_DIR=${APP_DIR:-/opt/codex-app-server} +BACKUP_DIR=${BACKUP_DIR:-/var/lib/codex-app-server/backups} + +mkdir -p "$BACKUP_DIR" +chmod 700 "$BACKUP_DIR" +cd "$APP_DIR" +python3 -m cloud.backup --backend postgres --out "$BACKUP_DIR" --keep 7