host
64995fafab
- migrations: MIGRATION_TABLES now equals POSTGRES_SCHEMA tables
(uploads, task_attachments, task_followups, environments added);
SEQUENCE_TABLES matches IDENTITY tables; task_followups
sender_task_id column ensured before insert; UNMIGRATED_TABLES
and migration_coverage() report tables with no PG schema instead
of silently skipping them; CLI prints the coverage report.
- deploy: DEFAULT_TEST_SUITES gains deploy/tests (discover -s only,
never package path); regression tests pin the default list.
- monitor: MONITOR_SINGLE_HOST=1 makes the server check single-host;
default stays dual to avoid silent downgrade; server_mode is in
the JSON. Optional fail-only alert outlet (webhook or mail,
default off, alert body is pass/fail JSON only, no secrets).
- backup/rollback: README and restore-drill document what is
recoverable (DB, code) vs not (worktrees, traces, uploads, host
configs); rollback.sh shares the server-unit list and dual-port
health success standard with host_deploy.sh (sync test guards it).
- public entry: Caddyfile uses native {$DOMAIN}/{$ACME_EMAIL}
variables plus a caddy.service template; README documents the
steps and the current BT-nginx production status.
259 lines
9.9 KiB
Bash
Executable File
259 lines
9.9 KiB
Bash
Executable File
#!/bin/sh
|
||
# 主机侧监控检查：输出一行 JSON 状态，任一项 fail 则退出非零。
|
||
#
|
||
# 用法（主机上，codexsvc 可读权限即可）：
|
||
# sh /opt/codex-app-server/deploy/monitor.sh
|
||
# sh /opt/codex-app-server/deploy/monitor.sh --self-test
|
||
#
|
||
# 检查项（全部不需要密码、不读 env 文件）：
|
||
# server：systemctl is-active codex-cloud-server 与
|
||
# codex-cloud-server-2 均为 active（HA 阶段 2 双机）
|
||
# worker：codex-autoscaler 或 codex-cloud-worker 任一为 active
|
||
# （autoscaler 接管后静态 worker 故意停用，只查静态会误报：
|
||
# 2026-10-26 07:02:55 静态 worker 停用后每 5 分钟 worker fail，
|
||
# 实际 worker 正常、health 的 worker_ok 也一直 pass）
|
||
# health：curl health，要求 ok=true、backend=postgres、db_ok=true、
|
||
# worker_ok=true（worker 心跳 180 秒内，见 server.py）
|
||
# postgres：pg_isready -h 127.0.0.1 -p 5432（只测就绪，不连库查数据，
|
||
# 所以不需要密码；需要密码的 SELECT 1 那一步直接跳过，不做）
|
||
# docker_postgres：docker ps 里 codex-postgres 在运行
|
||
# disk：磁盘使用率 < 90%（默认看 /var/lib/codex-app-server，不存在则看 /）
|
||
# app_dir：/opt/codex-app-server 目录存在
|
||
# backup：最近的 cloud-backup-*.sql 在 36 小时内（每天一次定时备份，
|
||
# HA 阶段 0；目录默认 /var/lib/codex-app-server/backups，可用
|
||
# MONITOR_BACKUP_DIR 覆盖）
|
||
#
|
||
# 输出示例（单行 JSON，值只会是 pass/fail，不含任何密钥）：
|
||
# {"overall": "pass", "checks": {"server": "pass", ...}}
|
||
#
|
||
# --self-test：只验证脚本本身的机制（health JSON 解析 + 输出是合法 JSON），
|
||
# 不验证主机健康。它会起一个本地 canned health server（只返回固定健康
|
||
# JSON），断言 health 那一项为 pass；其它项在非主机机器上 fail 是正常的，
|
||
# self-test 不因此失败。不要把 --self-test 通过误读成“主机健康”。
|
||
#
|
||
# 单机模式（P2-ops）：默认 server 检查要求 server-1 与 server-2
|
||
# 双 active（HA 阶段 2），单机主机上 server-2 不存在会恒 fail，
|
||
# 恒 fail 会掩盖真故障。确认是单机部署时设 MONITOR_SINGLE_HOST=1，
|
||
# server 检查只查 server-1；双机主机上绝不要设它（设了就是
|
||
# 静默降级，冗余丢了也看不见）。输出 JSON 里 server_mode 字段
|
||
# 写明本次用的是 single 还是 dual，降级可见。
|
||
#
|
||
# 告警出口（P2-ops，默认关闭）：只在 overall=fail 时发一次，
|
||
# pass 时不发；故障持续期间 timer 每 5 分钟跑一次就会重发，
|
||
# 接收端需自行去重（本机不做去重状态文件，免得状态文件本身
|
||
# 成为新的故障点）。二选一，同时配了只用 webhook：
|
||
# MONITOR_ALERT_URL=https://example.invalid/hook # webhook POST JSON
|
||
# MONITOR_ALERT_MAILTO=ops@example.invalid # mail 命令发邮件
|
||
# 告警内容只有 JSON 检查结果（pass/fail 与 server_mode），
|
||
# 不含 health 正文、不含任何环境变量、不含密钥；URL 与收件人
|
||
# 只走环境变量，不写进脚本、不进日志。
|
||
set -eu
|
||
|
||
SERVER_MODE=dual
|
||
if [ "${MONITOR_SINGLE_HOST:-0}" = "1" ]; then SERVER_MODE=single; fi
|
||
|
||
HEALTH_URL=${MONITOR_HEALTH_URL:-http://127.0.0.1:8765/health}
|
||
APP_DIR=${MONITOR_APP_DIR:-/opt/codex-app-server}
|
||
DISK_PATH=${MONITOR_DISK_PATH:-/var/lib/codex-app-server}
|
||
if [ ! -d "$DISK_PATH" ]; then DISK_PATH="/"; fi
|
||
|
||
check_server() {
|
||
# 默认双机：两台 server 都必须 active，任一挂了都算 fail
|
||
# （单台 active 时 API 还能用，但冗余已丢，必须可见）。
|
||
# MONITOR_SINGLE_HOST=1 时只查 server-1（单机部署专用，
|
||
# 双机主机上设它等于静默降级，见文件头注释）。
|
||
if [ "$SERVER_MODE" = "single" ]; then
|
||
if systemctl is-active codex-cloud-server >/dev/null 2>&1; then
|
||
echo pass
|
||
else
|
||
echo fail
|
||
fi
|
||
return
|
||
fi
|
||
if systemctl is-active codex-cloud-server >/dev/null 2>&1 \
|
||
&& systemctl is-active codex-cloud-server-2 >/dev/null 2>&1; then
|
||
echo pass
|
||
else
|
||
echo fail
|
||
fi
|
||
}
|
||
|
||
check_worker() {
|
||
if systemctl is-active codex-autoscaler >/dev/null 2>&1 \
|
||
|| systemctl is-active codex-cloud-worker >/dev/null 2>&1; then
|
||
echo pass
|
||
else
|
||
echo fail
|
||
fi
|
||
}
|
||
|
||
check_health() {
|
||
OUT=$(curl -s --max-time 5 "$HEALTH_URL" || true)
|
||
if [ -z "$OUT" ]; then echo fail; return; fi
|
||
if printf '%s' "$OUT" | HEALTH_JSON="$OUT" python3 -c '
|
||
import json, os, sys
|
||
try:
|
||
d = json.loads(os.environ["HEALTH_JSON"])
|
||
except Exception:
|
||
sys.exit(1)
|
||
sys.exit(0 if (d.get("ok") is True and d.get("backend") == "postgres"
|
||
and d.get("db_ok") is True
|
||
and d.get("worker_ok") is True) else 1)
|
||
' 2>/dev/null; then
|
||
echo pass
|
||
else
|
||
echo fail
|
||
fi
|
||
}
|
||
|
||
check_postgres() {
|
||
if command -v pg_isready >/dev/null 2>&1 \
|
||
&& pg_isready -h 127.0.0.1 -p 5432 >/dev/null 2>&1; then
|
||
echo pass
|
||
else
|
||
echo fail
|
||
fi
|
||
}
|
||
|
||
check_docker_postgres() {
|
||
if command -v docker >/dev/null 2>&1 \
|
||
&& docker ps --format '{{.Names}}' 2>/dev/null \
|
||
| grep -qx 'codex-postgres'; then
|
||
echo pass
|
||
else
|
||
echo fail
|
||
fi
|
||
}
|
||
|
||
check_disk() {
|
||
USE=$(df -P "$DISK_PATH" 2>/dev/null | awk 'NR==2 {print $5}' \
|
||
| tr -d '% ' || true)
|
||
case "$USE" in
|
||
''|*[!0-9]*) echo fail ;;
|
||
*) if [ "$USE" -lt 90 ]; then echo pass; else echo fail; fi ;;
|
||
esac
|
||
}
|
||
|
||
check_app_dir() {
|
||
if [ -d "$APP_DIR" ]; then echo pass; else echo fail; fi
|
||
}
|
||
|
||
check_backup() {
|
||
# 备份新鲜度：BACKUP_DIR 里最近的 cloud-backup-*.sql 在 36 小时内
|
||
# （每天一次，36 小时容忍一次失败 + 时区/夏令时抖动）。目录或文件
|
||
# 缺失一律 fail——备份制度断了必须在 monitor 里可见，不能静默。
|
||
BACKUP_DIR=${MONITOR_BACKUP_DIR:-/var/lib/codex-app-server/backups}
|
||
NEWEST=$(ls -t "$BACKUP_DIR"/cloud-backup-*.sql 2>/dev/null | head -1 || true)
|
||
if [ -z "$NEWEST" ]; then echo fail; return; fi
|
||
MTIME=$(stat -c %Y "$NEWEST" 2>/dev/null || true)
|
||
case "$MTIME" in
|
||
''|*[!0-9]*) echo fail; return ;;
|
||
esac
|
||
NOW=$(date +%s)
|
||
AGE=$((NOW - MTIME))
|
||
if [ "$AGE" -lt 129600 ]; then echo pass; else echo fail; fi
|
||
}
|
||
|
||
run_checks() {
|
||
S=$(check_server); W=$(check_worker); H=$(check_health)
|
||
P=$(check_postgres); D=$(check_docker_postgres)
|
||
K=$(check_disk); A=$(check_app_dir); B=$(check_backup)
|
||
OVERALL=pass
|
||
for v in "$S" "$W" "$H" "$P" "$D" "$K" "$A" "$B"; do
|
||
if [ "$v" != "pass" ]; then OVERALL=fail; break; fi
|
||
done
|
||
LAST_JSON=$(printf '{"overall": "%s", "server_mode": "%s", "checks": {"server": "%s", "worker": "%s", "health": "%s", "postgres": "%s", "docker_postgres": "%s", "disk": "%s", "app_dir": "%s", "backup": "%s"}}' \
|
||
"$OVERALL" "$SERVER_MODE" "$S" "$W" "$H" "$P" "$D" "$K" "$A" "$B")
|
||
printf '%s\n' "$LAST_JSON"
|
||
if [ "$OVERALL" = "pass" ]; then return 0; else return 1; fi
|
||
}
|
||
|
||
send_alert() {
|
||
# 只在 fail 时由主流程调用。内容就是 LAST_JSON（pass/fail
|
||
# 与 server_mode），绝不拼 health 正文、环境变量或密钥。
|
||
if [ -n "${MONITOR_ALERT_URL:-}" ]; then
|
||
if [ -n "${MONITOR_ALERT_MAILTO:-}" ]; then
|
||
echo "monitor: webhook 与邮件同时配置，只用 webhook" >&2
|
||
fi
|
||
curl -s --max-time 10 -X POST -H 'Content-Type: application/json' \
|
||
-d "$LAST_JSON" "$MONITOR_ALERT_URL" >/dev/null 2>&1 \
|
||
|| echo "monitor: webhook 告警发送失败" >&2
|
||
return 0
|
||
fi
|
||
if [ -n "${MONITOR_ALERT_MAILTO:-}" ]; then
|
||
if command -v mail >/dev/null 2>&1; then
|
||
printf '%s\n' "$LAST_JSON" \
|
||
| mail -s "codex monitor FAIL" "$MONITOR_ALERT_MAILTO" \
|
||
2>/dev/null \
|
||
|| echo "monitor: 邮件告警发送失败" >&2
|
||
else
|
||
echo "monitor: 已配 MAILTO 但本机无 mail 命令，告警未发出" >&2
|
||
fi
|
||
fi
|
||
}
|
||
|
||
if [ "${1:-}" = "--self-test" ]; then
|
||
# 起一个 canned health server：只返回固定健康 JSON，不读任何真实服务。
|
||
SELF_PORT=18999
|
||
python3 - "$SELF_PORT" <<'PY' &
|
||
import http.server
|
||
import json
|
||
import sys
|
||
|
||
|
||
class H(http.server.BaseHTTPRequestHandler):
|
||
def do_GET(self):
|
||
body = json.dumps({
|
||
"ok": True, "service": "cloud-control-plane",
|
||
"backend": "postgres", "db_ok": True,
|
||
"worker_last_seen_seconds_ago": 1.0, "worker_ok": True,
|
||
}).encode()
|
||
|
||
self.send_response(200)
|
||
self.send_header("Content-Type", "application/json")
|
||
self.send_header("Content-Length", str(len(body)))
|
||
self.end_headers()
|
||
self.wfile.write(body)
|
||
|
||
def log_message(self, *a):
|
||
pass
|
||
|
||
|
||
http.server.HTTPServer(("127.0.0.1", int(sys.argv[1])), H).serve_forever()
|
||
PY
|
||
SERVER_PID=$!
|
||
trap 'kill $SERVER_PID 2>/dev/null || true' EXIT
|
||
sleep 1
|
||
# 内层跑完整检查时清掉告警出口：self-test 在非主机上
|
||
# 其它项必然 fail，不清会在配了告警的环境里误发一条。
|
||
MONITOR_HEALTH_URL="http://127.0.0.1:$SELF_PORT/health" \
|
||
HEALTH_URL="http://127.0.0.1:$SELF_PORT/health" \
|
||
MONITOR_ALERT_URL= MONITOR_ALERT_MAILTO= \
|
||
sh "$0" > /tmp/monitor-self-test.json 2>/dev/null || true
|
||
# 注意：上面那行是跑完整检查，health 以外项在非主机上 fail 是正常的；
|
||
# self-test 只断言两件事：输出是合法 JSON，且 health 解析为 pass。
|
||
if python3 -c '
|
||
import json
|
||
import sys
|
||
with open("/tmp/monitor-self-test.json") as f:
|
||
d = json.load(f)
|
||
sys.exit(0 if d["checks"]["health"] == "pass" else 1)
|
||
'; then
|
||
echo "SELF-TEST PASS：输出是合法 JSON 且 health 解析为 pass（其它项在非主机上 fail 属正常，不影响 self-test）"
|
||
cat /tmp/monitor-self-test.json
|
||
exit 0
|
||
else
|
||
echo "SELF-TEST FAIL：health 解析未通过或输出不是合法 JSON" >&2
|
||
cat /tmp/monitor-self-test.json >&2 || true
|
||
exit 1
|
||
fi
|
||
fi
|
||
|
||
if run_checks; then
|
||
exit 0
|
||
fi
|
||
# 只有 fail 走到这里：默认不配出口时 send_alert 是空操作，
|
||
# 不会因为没配告警就多报错；配了出口则发一条 fail 告警。
|
||
send_alert
|
||
exit 1
|