P2-ops: migration alignment, deploy test list, monitor alerts/single-host, backup/rollback closure, Caddy template
- migrations: MIGRATION_TABLES now equals POSTGRES_SCHEMA tables
(uploads, task_attachments, task_followups, environments added);
SEQUENCE_TABLES matches IDENTITY tables; task_followups
sender_task_id column ensured before insert; UNMIGRATED_TABLES
and migration_coverage() report tables with no PG schema instead
of silently skipping them; CLI prints the coverage report.
- deploy: DEFAULT_TEST_SUITES gains deploy/tests (discover -s only,
never package path); regression tests pin the default list.
- monitor: MONITOR_SINGLE_HOST=1 makes the server check single-host;
default stays dual to avoid silent downgrade; server_mode is in
the JSON. Optional fail-only alert outlet (webhook or mail,
default off, alert body is pass/fail JSON only, no secrets).
- backup/rollback: README and restore-drill document what is
recoverable (DB, code) vs not (worktrees, traces, uploads, host
configs); rollback.sh shares the server-unit list and dual-port
health success standard with host_deploy.sh (sync test guards it).
- public entry: Caddyfile uses native {$DOMAIN}/{$ACME_EMAIL}
variables plus a caddy.service template; README documents the
steps and the current BT-nginx production status.
This commit is contained in:
host committed
1 parent
ef532a38c2
commit
64995fafab
14 files changed
+953
-44
No files matched your search
+67
-7
@@ -31,19 +31,45 @@
|
||||
# JSON），断言 health 那一项为 pass；其它项在非主机机器上 fail 是正常的，
|
||||
# self-test 不因此失败。不要把 --self-test 通过误读成“主机健康”。
|
||||
#
|
||||
# 告警边界：本脚本只做“检查 + journal 日志”（配合 monitor.timer 每 5 分钟
|
||||
# 跑一次），它本身不会发消息给用户。推送到 Muse 聊天的告警仍需用户另行
|
||||
# 决定通道（SSH key 常驻通道或 webhook URL），见 deploy/README.md。
|
||||
# 单机模式（P2-ops）：默认 server 检查要求 server-1 与 server-2
|
||||
# 双 active（HA 阶段 2），单机主机上 server-2 不存在会恒 fail，
|
||||
# 恒 fail 会掩盖真故障。确认是单机部署时设 MONITOR_SINGLE_HOST=1，
|
||||
# server 检查只查 server-1；双机主机上绝不要设它（设了就是
|
||||
# 静默降级，冗余丢了也看不见）。输出 JSON 里 server_mode 字段
|
||||
# 写明本次用的是 single 还是 dual，降级可见。
|
||||
#
|
||||
# 告警出口（P2-ops，默认关闭）：只在 overall=fail 时发一次，
|
||||
# pass 时不发；故障持续期间 timer 每 5 分钟跑一次就会重发，
|
||||
# 接收端需自行去重（本机不做去重状态文件，免得状态文件本身
|
||||
# 成为新的故障点）。二选一，同时配了只用 webhook：
|
||||
# MONITOR_ALERT_URL=https://example.invalid/hook # webhook POST JSON
|
||||
# MONITOR_ALERT_MAILTO=ops@example.invalid # mail 命令发邮件
|
||||
# 告警内容只有 JSON 检查结果（pass/fail 与 server_mode），
|
||||
# 不含 health 正文、不含任何环境变量、不含密钥；URL 与收件人
|
||||
# 只走环境变量，不写进脚本、不进日志。
|
||||
set -eu
|
||||
|
||||
SERVER_MODE=dual
|
||||
if [ "${MONITOR_SINGLE_HOST:-0}" = "1" ]; then SERVER_MODE=single; fi
|
||||
|
||||
HEALTH_URL=${MONITOR_HEALTH_URL:-http://127.0.0.1:8765/health}
|
||||
APP_DIR=${MONITOR_APP_DIR:-/opt/codex-app-server}
|
||||
DISK_PATH=${MONITOR_DISK_PATH:-/var/lib/codex-app-server}
|
||||
if [ ! -d "$DISK_PATH" ]; then DISK_PATH="/"; fi
|
||||
|
||||
check_server() {
|
||||
# HA 阶段 2：两台 server 都必须 active，任一挂了都算 fail
|
||||
# 默认双机：两台 server 都必须 active，任一挂了都算 fail
|
||||
# （单台 active 时 API 还能用，但冗余已丢，必须可见）。
|
||||
# MONITOR_SINGLE_HOST=1 时只查 server-1（单机部署专用，
|
||||
# 双机主机上设它等于静默降级，见文件头注释）。
|
||||
if [ "$SERVER_MODE" = "single" ]; then
|
||||
if systemctl is-active codex-cloud-server >/dev/null 2>&1; then
|
||||
echo pass
|
||||
else
|
||||
echo fail
|
||||
fi
|
||||
return
|
||||
fi
|
||||
if systemctl is-active codex-cloud-server >/dev/null 2>&1 \
|
||||
&& systemctl is-active codex-cloud-server-2 >/dev/null 2>&1; then
|
||||
echo pass
|
||||
@@ -136,11 +162,36 @@ run_checks() {
|
||||
for v in "$S" "$W" "$H" "$P" "$D" "$K" "$A" "$B"; do
|
||||
if [ "$v" != "pass" ]; then OVERALL=fail; break; fi
|
||||
done
|
||||
printf '{"overall": "%s", "checks": {"server": "%s", "worker": "%s", "health": "%s", "postgres": "%s", "docker_postgres": "%s", "disk": "%s", "app_dir": "%s", "backup": "%s"}}\n' \
|
||||
"$OVERALL" "$S" "$W" "$H" "$P" "$D" "$K" "$A" "$B"
|
||||
LAST_JSON=$(printf '{"overall": "%s", "server_mode": "%s", "checks": {"server": "%s", "worker": "%s", "health": "%s", "postgres": "%s", "docker_postgres": "%s", "disk": "%s", "app_dir": "%s", "backup": "%s"}}' \
|
||||
"$OVERALL" "$SERVER_MODE" "$S" "$W" "$H" "$P" "$D" "$K" "$A" "$B")
|
||||
printf '%s\n' "$LAST_JSON"
|
||||
if [ "$OVERALL" = "pass" ]; then return 0; else return 1; fi
|
||||
}
|
||||
|
||||
send_alert() {
|
||||
# 只在 fail 时由主流程调用。内容就是 LAST_JSON（pass/fail
|
||||
# 与 server_mode），绝不拼 health 正文、环境变量或密钥。
|
||||
if [ -n "${MONITOR_ALERT_URL:-}" ]; then
|
||||
if [ -n "${MONITOR_ALERT_MAILTO:-}" ]; then
|
||||
echo "monitor: webhook 与邮件同时配置，只用 webhook" >&2
|
||||
fi
|
||||
curl -s --max-time 10 -X POST -H 'Content-Type: application/json' \
|
||||
-d "$LAST_JSON" "$MONITOR_ALERT_URL" >/dev/null 2>&1 \
|
||||
|| echo "monitor: webhook 告警发送失败" >&2
|
||||
return 0
|
||||
fi
|
||||
if [ -n "${MONITOR_ALERT_MAILTO:-}" ]; then
|
||||
if command -v mail >/dev/null 2>&1; then
|
||||
printf '%s\n' "$LAST_JSON" \
|
||||
| mail -s "codex monitor FAIL" "$MONITOR_ALERT_MAILTO" \
|
||||
2>/dev/null \
|
||||
|| echo "monitor: 邮件告警发送失败" >&2
|
||||
else
|
||||
echo "monitor: 已配 MAILTO 但本机无 mail 命令，告警未发出" >&2
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
if [ "${1:-}" = "--self-test" ]; then
|
||||
# 起一个 canned health server：只返回固定健康 JSON，不读任何真实服务。
|
||||
SELF_PORT=18999
|
||||
@@ -173,8 +224,11 @@ PY
|
||||
SERVER_PID=$!
|
||||
trap 'kill $SERVER_PID 2>/dev/null || true' EXIT
|
||||
sleep 1
|
||||
# 内层跑完整检查时清掉告警出口：self-test 在非主机上
|
||||
# 其它项必然 fail，不清会在配了告警的环境里误发一条。
|
||||
MONITOR_HEALTH_URL="http://127.0.0.1:$SELF_PORT/health" \
|
||||
HEALTH_URL="http://127.0.0.1:$SELF_PORT/health" \
|
||||
MONITOR_ALERT_URL= MONITOR_ALERT_MAILTO= \
|
||||
sh "$0" > /tmp/monitor-self-test.json 2>/dev/null || true
|
||||
# 注意：上面那行是跑完整检查，health 以外项在非主机上 fail 是正常的；
|
||||
# self-test 只断言两件事：输出是合法 JSON，且 health 解析为 pass。
|
||||
@@ -195,4 +249,10 @@ sys.exit(0 if d["checks"]["health"] == "pass" else 1)
|
||||
fi
|
||||
fi
|
||||
|
||||
run_checks
|
||||
if run_checks; then
|
||||
exit 0
|
||||
fi
|
||||
# 只有 fail 走到这里：默认不配出口时 send_alert 是空操作，
|
||||
# 不会因为没配告警就多报错；配了出口则发一条 fail 告警。
|
||||
send_alert
|
||||
exit 1
|
||||
Reference in new issue
Block a user