Ops batch: deploy automation, backup restore drill, monitoring, multi-user verification
Deploy: deploy.sh builds tarball, backs up current code (keep 3), deploys, restarts, health-gates 60s (ok/backend=postgres/db_ok), auto-rolls back on failure; rollback.sh restores latest backup. Rollback is code-only, DB migrations are additive-only so old code can still read new columns. Backup: restore_postgres refuses to restore into the production DB name unless explicitly allowed (drill never allows it); restore-drill.sh backs up fresh, restores to a test DB, compares tasks/projects/users/audit row counts, drops the test DB. Monitoring: worker writes heartbeat file every loop (atomic replace, failure only logged); GET /health adds worker_last_seen_seconds_ago + worker_ok (180s default, env tunable, still HTTP 200 when worker is stale); monitor.sh checks server/worker/health/pg_isready/docker-postgres/disk/ app dir as one JSON line, exit non-zero on any fail, plus systemd timer every 5 minutes. Push alerting to chat still needs a separate channel decision (SSH key or webhook URL). Multi-user: verify-multiuser.py creates test users/projects in memory-key only, checks RBAC via real HTTP API (owner/ maintainer/contributor/viewer, cross-project isolation, merge 200/403/409, no-key 401), test tasks carry a never- satisfiable dependency so the production worker never claims them, cleanup deletes only multiuser-test- rows (audit log is append-only and keeps test entries by design). Tests: host 27 pass, cloud 298 pass (8 skipped are environment-dependent real-Postgres tests).
This commit is contained in:
host committed
1 parent
6afef78ca8
commit
89a918b543
14 files changed
+1386
-2
No files matched your search
Executable
+171
@@ -0,0 +1,171 @@
|
||||
#!/bin/sh
|
||||
# 主机侧监控检查：输出一行 JSON 状态，任一项 fail 则退出非零。
|
||||
#
|
||||
# 用法（主机上，codexsvc 可读权限即可）：
|
||||
# sh /opt/codex-app-server/deploy/monitor.sh
|
||||
# sh /opt/codex-app-server/deploy/monitor.sh --self-test
|
||||
#
|
||||
# 检查项（全部不需要密码、不读 env 文件）：
|
||||
# server：systemctl is-active codex-cloud-server 为 active
|
||||
# worker：systemctl is-active codex-cloud-worker 为 active
|
||||
# health：curl health，要求 ok=true、backend=postgres、db_ok=true、
|
||||
# worker_ok=true（worker 心跳 180 秒内，见 server.py）
|
||||
# postgres：pg_isready -h 127.0.0.1 -p 5432（只测就绪，不连库查数据，
|
||||
# 所以不需要密码；需要密码的 SELECT 1 那一步直接跳过，不做）
|
||||
# docker_postgres：docker ps 里 codex-postgres 在运行
|
||||
# disk：磁盘使用率 < 90%（默认看 /var/lib/codex-app-server，不存在则看 /）
|
||||
# app_dir：/opt/codex-app-server 目录存在
|
||||
#
|
||||
# 输出示例（单行 JSON，值只会是 pass/fail，不含任何密钥）：
|
||||
# {"overall": "pass", "checks": {"server": "pass", ...}}
|
||||
#
|
||||
# --self-test：只验证脚本本身的机制（health JSON 解析 + 输出是合法 JSON），
|
||||
# 不验证主机健康。它会起一个本地 canned health server（只返回固定健康
|
||||
# JSON），断言 health 那一项为 pass；其它项在非主机机器上 fail 是正常的，
|
||||
# self-test 不因此失败。不要把 --self-test 通过误读成“主机健康”。
|
||||
#
|
||||
# 告警边界：本脚本只做“检查 + journal 日志”（配合 monitor.timer 每 5 分钟
|
||||
# 跑一次），它本身不会发消息给用户。推送到 Muse 聊天的告警仍需用户另行
|
||||
# 决定通道（SSH key 常驻通道或 webhook URL），见 deploy/README.md。
|
||||
set -eu
|
||||
|
||||
HEALTH_URL=${MONITOR_HEALTH_URL:-http://127.0.0.1:8765/health}
|
||||
APP_DIR=${MONITOR_APP_DIR:-/opt/codex-app-server}
|
||||
DISK_PATH=${MONITOR_DISK_PATH:-/var/lib/codex-app-server}
|
||||
if [ ! -d "$DISK_PATH" ]; then DISK_PATH="/"; fi
|
||||
|
||||
check_server() {
|
||||
if systemctl is-active codex-cloud-server >/dev/null 2>&1; then
|
||||
echo pass
|
||||
else
|
||||
echo fail
|
||||
fi
|
||||
}
|
||||
|
||||
check_worker() {
|
||||
if systemctl is-active codex-cloud-worker >/dev/null 2>&1; then
|
||||
echo pass
|
||||
else
|
||||
echo fail
|
||||
fi
|
||||
}
|
||||
|
||||
check_health() {
|
||||
OUT=$(curl -s --max-time 5 "$HEALTH_URL" || true)
|
||||
if [ -z "$OUT" ]; then echo fail; return; fi
|
||||
if printf '%s' "$OUT" | HEALTH_JSON="$OUT" python3 -c '
|
||||
import json, os, sys
|
||||
try:
|
||||
d = json.loads(os.environ["HEALTH_JSON"])
|
||||
except Exception:
|
||||
sys.exit(1)
|
||||
sys.exit(0 if (d.get("ok") is True and d.get("backend") == "postgres"
|
||||
and d.get("db_ok") is True
|
||||
and d.get("worker_ok") is True) else 1)
|
||||
' 2>/dev/null; then
|
||||
echo pass
|
||||
else
|
||||
echo fail
|
||||
fi
|
||||
}
|
||||
|
||||
check_postgres() {
|
||||
if command -v pg_isready >/dev/null 2>&1 \
|
||||
&& pg_isready -h 127.0.0.1 -p 5432 >/dev/null 2>&1; then
|
||||
echo pass
|
||||
else
|
||||
echo fail
|
||||
fi
|
||||
}
|
||||
|
||||
check_docker_postgres() {
|
||||
if command -v docker >/dev/null 2>&1 \
|
||||
&& docker ps --format '{{.Names}}' 2>/dev/null \
|
||||
| grep -qx 'codex-postgres'; then
|
||||
echo pass
|
||||
else
|
||||
echo fail
|
||||
fi
|
||||
}
|
||||
|
||||
check_disk() {
|
||||
USE=$(df -P "$DISK_PATH" 2>/dev/null | awk 'NR==2 {print $5}' \
|
||||
| tr -d '% ' || true)
|
||||
case "$USE" in
|
||||
''|*[!0-9]*) echo fail ;;
|
||||
*) if [ "$USE" -lt 90 ]; then echo pass; else echo fail; fi ;;
|
||||
esac
|
||||
}
|
||||
|
||||
check_app_dir() {
|
||||
if [ -d "$APP_DIR" ]; then echo pass; else echo fail; fi
|
||||
}
|
||||
|
||||
run_checks() {
|
||||
S=$(check_server); W=$(check_worker); H=$(check_health)
|
||||
P=$(check_postgres); D=$(check_docker_postgres)
|
||||
K=$(check_disk); A=$(check_app_dir)
|
||||
OVERALL=pass
|
||||
for v in "$S" "$W" "$H" "$P" "$D" "$K" "$A"; do
|
||||
if [ "$v" != "pass" ]; then OVERALL=fail; break; fi
|
||||
done
|
||||
printf '{"overall": "%s", "checks": {"server": "%s", "worker": "%s", "health": "%s", "postgres": "%s", "docker_postgres": "%s", "disk": "%s", "app_dir": "%s"}}\n' \
|
||||
"$OVERALL" "$S" "$W" "$H" "$P" "$D" "$K" "$A"
|
||||
if [ "$OVERALL" = "pass" ]; then return 0; else return 1; fi
|
||||
}
|
||||
|
||||
if [ "${1:-}" = "--self-test" ]; then
|
||||
# 起一个 canned health server：只返回固定健康 JSON，不读任何真实服务。
|
||||
SELF_PORT=18999
|
||||
python3 - "$SELF_PORT" <<'PY' &
|
||||
import http.server
|
||||
import json
|
||||
import sys
|
||||
|
||||
|
||||
class H(http.server.BaseHTTPRequestHandler):
|
||||
def do_GET(self):
|
||||
body = json.dumps({
|
||||
"ok": True, "service": "cloud-control-plane",
|
||||
"backend": "postgres", "db_ok": True,
|
||||
"worker_last_seen_seconds_ago": 1.0, "worker_ok": True,
|
||||
}).encode()
|
||||
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def log_message(self, *a):
|
||||
pass
|
||||
|
||||
|
||||
http.server.HTTPServer(("127.0.0.1", int(sys.argv[1])), H).serve_forever()
|
||||
PY
|
||||
SERVER_PID=$!
|
||||
trap 'kill $SERVER_PID 2>/dev/null || true' EXIT
|
||||
sleep 1
|
||||
MONITOR_HEALTH_URL="http://127.0.0.1:$SELF_PORT/health" \
|
||||
HEALTH_URL="http://127.0.0.1:$SELF_PORT/health" \
|
||||
sh "$0" > /tmp/monitor-self-test.json 2>/dev/null || true
|
||||
# 注意：上面那行是跑完整检查，health 以外项在非主机上 fail 是正常的；
|
||||
# self-test 只断言两件事：输出是合法 JSON，且 health 解析为 pass。
|
||||
if python3 -c '
|
||||
import json
|
||||
import sys
|
||||
with open("/tmp/monitor-self-test.json") as f:
|
||||
d = json.load(f)
|
||||
sys.exit(0 if d["checks"]["health"] == "pass" else 1)
|
||||
'; then
|
||||
echo "SELF-TEST PASS：输出是合法 JSON 且 health 解析为 pass（其它项在非主机上 fail 属正常，不影响 self-test）"
|
||||
cat /tmp/monitor-self-test.json
|
||||
exit 0
|
||||
else
|
||||
echo "SELF-TEST FAIL：health 解析未通过或输出不是合法 JSON" >&2
|
||||
cat /tmp/monitor-self-test.json >&2 || true
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
run_checks
|
||||
Reference in new issue
Block a user