Ops batch: deploy automation, backup restore drill, monitoring, multi-user verification

Deploy: deploy.sh builds tarball, backs up current code (keep 3),
deploys, restarts, health-gates 60s (ok/backend=postgres/db_ok),
auto-rolls back on failure; rollback.sh restores latest backup.
Rollback is code-only, DB migrations are additive-only so old
code can still read new columns.
Backup: restore_postgres refuses to restore into the production
DB name unless explicitly allowed (drill never allows it);
restore-drill.sh backs up fresh, restores to a test DB, compares
tasks/projects/users/audit row counts, drops the test DB.
Monitoring: worker writes heartbeat file every loop (atomic
replace, failure only logged); GET /health adds
worker_last_seen_seconds_ago + worker_ok (180s default, env
tunable, still HTTP 200 when worker is stale); monitor.sh
checks server/worker/health/pg_isready/docker-postgres/disk/
app dir as one JSON line, exit non-zero on any fail, plus
systemd timer every 5 minutes. Push alerting to chat still
needs a separate channel decision (SSH key or webhook URL).
Multi-user: verify-multiuser.py creates test users/projects
in memory-key only, checks RBAC via real HTTP API (owner/
maintainer/contributor/viewer, cross-project isolation,
merge 200/403/409, no-key 401), test tasks carry a never-
satisfiable dependency so the production worker never claims
them, cleanup deletes only multiuser-test- rows (audit log
is append-only and keeps test entries by design).
Tests: host 27 pass, cloud 298 pass (8 skipped are
environment-dependent real-Postgres tests).

cloud/worker.py

@@ -208,7 +208,7 @@ class Worker:
                  cost_store=None, process_factory=None,
                  task_timeout=1800, max_retries=1,
                  retry_backoff_seconds=5.0, repos_dir="./repos",
-                 repos_runner=None):
+                 repos_runner=None, heartbeat_file=None):
         self.db_path = str(db_path)
         self.host_py_path = host_py_path
         self.config_path = config_path
@@ -252,6 +252,31 @@ class Worker:
         # repos_runner 可注入（单测用假 git runner），缺省走真实 git。
         self.repos_dir = repos_dir
         self.repos_runner = repos_runner
+        # 运维心跳：路径优先显式参数，其次环境变量，默认当前目录下
+        # 的 worker-heartbeat.json；主机路径不在代码里硬编码，主机
+        # 部署时用 --heartbeat-file 或环境变量传入（见 deploy/README）。
+        # 文件里只有 last_seen 与 pid，绝无任何密钥。
+        self.heartbeat_file = heartbeat_file or \
+            os.environ.get("CLOUD_WORKER_HEARTBEAT_FILE") or \
+            "./worker-heartbeat.json"
+
+    def _write_heartbeat(self):
+        """写一次运维心跳：写失败只记日志，绝不炸 worker。
+
+        先写临时文件再 os.replace，保证 server 读到的要么是旧的
+        完整文件、要么是新的完整文件，不会读到写一半的 JSON。
+        """
+        try:
+            payload = {"last_seen": time.time(), "pid": os.getpid()}
+            tmp = "%s.tmp" % self.heartbeat_file
+            parent = os.path.dirname(os.path.abspath(self.heartbeat_file))
+            os.makedirs(parent, exist_ok=True)
+            with open(tmp, "w", encoding="utf-8") as f:
+                f.write(json.dumps(payload))
+            os.replace(tmp, self.heartbeat_file)
+        except Exception as exc:
+            log_event(self.logger, "worker_heartbeat_failed",
+                      message="worker heartbeat write failed: %s" % exc)
 
     # -- 成本存储（惰性建表，避免只认领不执行时也建表）--------------
     @property
@@ -954,6 +979,9 @@ class Worker:
         # 常驻启动清理兜底：清掉上一轮残留的 codex- 容器，
         # 失败只记日志（见 cleanup_stale_containers），不炸循环。
         self.cleanup_stale_containers()
+        # 运维心跳：启动时先写一次，之后每轮结束再写，server 的
+        # /health 据此算 worker_ok（见 server.py 的健康检查）。
+        self._write_heartbeat()
         started = time.time()
         all_results = []
         try:
@@ -972,6 +1000,9 @@ class Worker:
                               message="worker_loop_error: %s" % exc)
                     results = []
                 all_results.extend(results)
+                # 每轮结束写心跳：哪怕本轮一个任务都没派，worker 活着
+                # 这件事本身也要被 server 看到。
+                self._write_heartbeat()
                 # 按 0.2 秒切片睡，期间可响应 stop_event/_loop_stop。
                 deadline = time.time() + interval
                 while time.time() < deadline:
@@ -1025,6 +1056,12 @@ def main(argv=None):
                         help="host.py 路径，缺省为 cloud/ 上一级的 host.py")
     parser.add_argument("--config", default=None)
     parser.add_argument("--repo", default=None)
+    parser.add_argument("--heartbeat-file", default=None,
+                        help="运维心跳文件路径（默认：环境变量 "
+                             "CLOUD_WORKER_HEARTBEAT_FILE，未设则为 "
+                             "./worker-heartbeat.json；主机部署时用此"
+                             "参数指到 /var/lib/codex-app-server/ 下，"
+                             "与 server 的 health 读取路径保持一致）")
     parser.add_argument("--repos-dir", default="./repos",
                         help="项目仓库检出根目录（默认 ./repos）；项目有"
                              " repo_url 时 worker 在此 clone/fetch 各项目"
@@ -1081,7 +1118,8 @@ def main(argv=None):
                     task_timeout=args.task_timeout,
                     max_retries=args.max_retries,
                     retry_backoff_seconds=args.retry_backoff_seconds,
-                    repos_dir=args.repos_dir)
+                    repos_dir=args.repos_dir,
+                    heartbeat_file=args.heartbeat_file)
     # CLI 启动清理兜底（常驻与单次都做）：清掉上一轮残留容器。
     worker.cleanup_stale_containers()
     # CLI 起 worker 前先建表/迁移：全新库直接起循环不应因缺表而炸。