P2-ops: migration alignment, deploy test list, monitor alerts/single-host, backup/rollback closure, Caddy template
- migrations: MIGRATION_TABLES now equals POSTGRES_SCHEMA tables
(uploads, task_attachments, task_followups, environments added);
SEQUENCE_TABLES matches IDENTITY tables; task_followups
sender_task_id column ensured before insert; UNMIGRATED_TABLES
and migration_coverage() report tables with no PG schema instead
of silently skipping them; CLI prints the coverage report.
- deploy: DEFAULT_TEST_SUITES gains deploy/tests (discover -s only,
never package path); regression tests pin the default list.
- monitor: MONITOR_SINGLE_HOST=1 makes the server check single-host;
default stays dual to avoid silent downgrade; server_mode is in
the JSON. Optional fail-only alert outlet (webhook or mail,
default off, alert body is pass/fail JSON only, no secrets).
- backup/rollback: README and restore-drill document what is
recoverable (DB, code) vs not (worktrees, traces, uploads, host
configs); rollback.sh shares the server-unit list and dual-port
health success standard with host_deploy.sh (sync test guards it).
- public entry: Caddyfile uses native {$DOMAIN}/{$ACME_EMAIL}
variables plus a caddy.service template; README documents the
steps and the current BT-nginx production status.
This commit is contained in:
host committed
1 parent
ef532a38c2
commit
64995fafab
14 files changed
+953
-44
No files matched your search
+123
-2
@@ -7,9 +7,22 @@
|
||||
- schema 来自 cloud.db.POSTGRES_SCHEMA / init_postgres_schema，
|
||||
本模块不另发明表结构；迁移前先建表（若目标库已有表，
|
||||
CREATE TABLE IF NOT EXISTS 是 no-op）。
|
||||
- P2-ops 对齐：MIGRATION_TABLES 必须等于 POSTGRES_SCHEMA 的
|
||||
表集合（见 migration_coverage 与 test_p2_migrations.py）。
|
||||
以前漏掉的 uploads、task_attachments、task_followups、
|
||||
environments 已补进清单；补进时发现 task_followups 在
|
||||
SQLite 侧比 PG schema 多一列 sender_task_id（peer_messages
|
||||
的运行时 ensure_followups_table 会在 PG 上 ALTER 补列，
|
||||
POSTGRES_SCHEMA 里没有它），迁移时按同样方式先补列再插，
|
||||
否则整批迁移会在该表上撞未知列失败。
|
||||
- kb_fts（SQLite FTS5 虚拟表）不迁移：Postgres 没有 FTS5，
|
||||
KBStore 在 PG 上走 LIKE 分支，kb_docs 的 content 已足够。
|
||||
- sqlite_sequence（AUTOINCREMENT 内部表）不迁移。
|
||||
- UNMIGRATED_TABLES：SQLite 各 store 建出、但 POSTGRES_SCHEMA
|
||||
里没有的表（api_keys、评审、自动化、GitHub App、网关 key
|
||||
等）。它们不是“被跳过”，而是本模块无法迁移——迁移报告
|
||||
（migration_coverage / CLI 输出）逐项列出，调用方不得
|
||||
把“迁移成功”读成“全部数据都过去了”。
|
||||
- 不打印、不返回任何密码或密钥：函数只返回 {表: inserted}，
|
||||
CLI 只打印表名与行数。
|
||||
- Postgres 连接参数只走 CLOUD_DB_* 环境变量（见 cloud.db），
|
||||
@@ -17,6 +30,7 @@
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
import sqlite3
|
||||
|
||||
from cloud.db import (
|
||||
@@ -43,10 +57,94 @@ MIGRATION_TABLES = (
|
||||
# （本批后无行按默认额度处理，不再是无限额，但显式
|
||||
# 管理员额度仍需原样迁过去，不能丢）。
|
||||
"user_quotas",
|
||||
# P2-ops 对齐：以下 4 张表在 POSTGRES_SCHEMA 里有、
|
||||
# 原清单里没有，补进后清单与 schema 表集合一致。
|
||||
# uploads / task_attachments：附件包与任务附件；
|
||||
# task_followups：任务间互发消息的后续消息队列；
|
||||
# environments：项目环境（分支/executor）。
|
||||
# 四张表主键均为 TEXT 或复合主键，没有身份列，
|
||||
# 不需要序列拨动（见 SEQUENCE_TABLES）。
|
||||
"uploads",
|
||||
"task_attachments",
|
||||
"task_followups",
|
||||
"environments",
|
||||
)
|
||||
|
||||
_SKIP_TABLES = frozenset({"kb_fts", "sqlite_sequence"})
|
||||
|
||||
# 身份列的表：POSTGRES_SCHEMA 里唯一用
|
||||
# ``BIGINT GENERATED BY DEFAULT AS IDENTITY`` 的 4 张表。
|
||||
# 迁移时显式插 id 不会推序列，迁完必须 setval 到 MAX(id)，
|
||||
# 否则后续省略 id 的 INSERT 会撞主键。新增身份列表必须
|
||||
# 同步改这里，test_p2_migrations.py 会拿 schema 实测对齐。
|
||||
SEQUENCE_TABLES = (
|
||||
"audit_log",
|
||||
"summaries",
|
||||
"kb_docs",
|
||||
"cost_records",
|
||||
)
|
||||
|
||||
# SQLite 各 store 建出、但 POSTGRES_SCHEMA 里没有的表：
|
||||
# 本迁移模块无法把它们迁到 Postgres（目标库无此表），
|
||||
# 不是静默跳过——migration_coverage() 与 CLI 逐项报告。
|
||||
# 其中 rate_limit_hits / workers 是易失的限流与心跳数据，
|
||||
# 丢了只影响限流窗口与 worker 可见性，不影响业务数据；
|
||||
# 其余（api_keys、task_reviews、automations、GitHub App、
|
||||
# user_gateway_keys 等）丢了就是真丢，迁库前必须另行
|
||||
# 处理（重建/重绑/重发 key），不能只跑本迁移就宣称迁完。
|
||||
UNMIGRATED_TABLES = (
|
||||
"api_keys",
|
||||
"task_approvals",
|
||||
"task_reviews",
|
||||
"automations",
|
||||
"automation_runs",
|
||||
"github_app_setup_states",
|
||||
"github_app_config",
|
||||
"github_app_installations",
|
||||
"github_fix_review_dedup",
|
||||
"external_identities",
|
||||
"pr_jobs",
|
||||
"rate_limit_hits",
|
||||
"user_gateway_keys",
|
||||
"webhook_deliveries",
|
||||
"workers",
|
||||
)
|
||||
|
||||
|
||||
def schema_tables():
|
||||
"""POSTGRES_SCHEMA 里的表名集合（只认 CREATE TABLE 语句）。"""
|
||||
names = set()
|
||||
for statement in POSTGRES_SCHEMA:
|
||||
m = re.search(r"CREATE TABLE IF NOT EXISTS (\w+)",
|
||||
statement)
|
||||
if m:
|
||||
names.add(m.group(1))
|
||||
return names
|
||||
|
||||
|
||||
def migration_coverage():
|
||||
"""迁移覆盖报告：清单、schema、未迁移项三者一次给全。
|
||||
|
||||
migrated：本次迁移模块会迁的表（= MIGRATION_TABLES）；
|
||||
schema_tables：POSTGRES_SCHEMA 里的表；
|
||||
missing_from_migration：schema 里有、清单里没有的表
|
||||
（本批后应为空；非空即对齐回归，调用方应报错
|
||||
而不是继续跑迁移）；
|
||||
unmigrated_no_pg_schema：UNMIGRATED_TABLES（目标库
|
||||
无此表，无法迁移，须另行处理）；
|
||||
skipped_by_design：kb_fts / sqlite_sequence（设计上不迁）。
|
||||
不含任何密钥或行内容，只含表名。
|
||||
"""
|
||||
schema = schema_tables()
|
||||
return {
|
||||
"migrated": tuple(MIGRATION_TABLES),
|
||||
"schema_tables": tuple(sorted(schema)),
|
||||
"missing_from_migration": tuple(
|
||||
sorted(schema - set(MIGRATION_TABLES))),
|
||||
"unmigrated_no_pg_schema": tuple(UNMIGRATED_TABLES),
|
||||
"skipped_by_design": tuple(sorted(_SKIP_TABLES)),
|
||||
}
|
||||
|
||||
|
||||
def migrate_sqlite_to_postgres(sqlite_path, pg_conn=None, tables=None):
|
||||
"""迁移并返回 {表名: 本次插入行数}。
|
||||
@@ -83,6 +181,26 @@ def migrate_sqlite_to_postgres(sqlite_path, pg_conn=None, tables=None):
|
||||
continue
|
||||
columns = [d[0] for d in src.execute(
|
||||
'SELECT * FROM "%s" LIMIT 0' % table).description]
|
||||
if table == "task_followups" and "sender_task_id" in columns:
|
||||
# SQLite 侧有、POSTGRES_SCHEMA 里没有的列：
|
||||
# 与 peer_messages.ensure_followups_table 同款，
|
||||
# 先查现有列、缺了再 ALTER 补列，已有列是 no-op。
|
||||
# PRAGMA 经适配层翻译成 information_schema 查询，
|
||||
# 取 row[1] 即列名（与 init_postgres_schema 同款）。
|
||||
# 补列失败也不在这里吞掉迁移：INSERT 时若列仍
|
||||
# 缺失会直接报错（大声失败，绝不静默丢列）。
|
||||
try:
|
||||
existing = {row[1] for row in pg_conn.execute(
|
||||
"PRAGMA table_info(task_followups)").fetchall()}
|
||||
except Exception:
|
||||
existing = set()
|
||||
if "sender_task_id" not in existing:
|
||||
try:
|
||||
pg_conn.execute(
|
||||
"ALTER TABLE task_followups"
|
||||
" ADD COLUMN sender_task_id TEXT")
|
||||
except Exception:
|
||||
pass
|
||||
placeholders = ", ".join("?" for _ in columns)
|
||||
col_list = ", ".join('"%s"' % c for c in columns)
|
||||
inserted = 0
|
||||
@@ -96,8 +214,7 @@ def migrate_sqlite_to_postgres(sqlite_path, pg_conn=None, tables=None):
|
||||
inserted += cur.rowcount
|
||||
# 身份列的序列不会因显式插 id 自动前进：迁完把序列
|
||||
# 拨到当前最大 id，否则后续省略 id 的 INSERT 会撞主键。
|
||||
if table in ("audit_log", "summaries", "kb_docs",
|
||||
"cost_records") and "id" in columns:
|
||||
if table in SEQUENCE_TABLES and "id" in columns:
|
||||
try:
|
||||
pg_conn.execute(
|
||||
"SELECT setval(pg_get_serial_sequence(?, ?),"
|
||||
@@ -120,5 +237,9 @@ def migrate_sqlite_to_postgres(sqlite_path, pg_conn=None, tables=None):
|
||||
|
||||
__all__ = [
|
||||
"MIGRATION_TABLES",
|
||||
"SEQUENCE_TABLES",
|
||||
"UNMIGRATED_TABLES",
|
||||
"migration_coverage",
|
||||
"migrate_sqlite_to_postgres",
|
||||
"schema_tables",
|
||||
]
|
||||
Reference in new issue
Block a user