host
dc3f572b2b
Replace the PostgresDatabase skeleton with a real implementation: psycopg (v3) first, psycopg2 second, lazy load, missing driver still raises PostgresDriverMissingError with no silent SQLite fallback. A sqlite3-style adapter (PgConnection/PgCursor/PgRow) translates ? placeholders to %s outside string literals, maps UNIQUE/FK violations, lock errors and connection failures to explicit Postgres error types, and handles INSERT OR REPLACE, PRAGMA table_info, AUTOINCREMENT and FTS5 refusal. POSTGRES_SCHEMA covers all business tables with tasks result columns; server init_db uses it under CLOUD_DB_BACKEND=postgres, SQLite path unchanged. migrate_sqlite_to_postgres.py copies rows with ON CONFLICT DO NOTHING and reseeds identity sequences. Tests: cloud 239 (235 pass, 4 skipped without a real server), host 27 pass; real-server tests skip unless CLOUD_TEST_PG_DSN is set.
121 lines
4.4 KiB
Python
121 lines
4.4 KiB
Python
"""SQLite -> Postgres 迁移（控制面状态库）。
|
||
|
||
只做一件事：把一个 SQLite 库文件里的行原样插进 Postgres，
|
||
表名与列名不改。幂等：Postgres 侧用 ``ON CONFLICT DO NOTHING``，
|
||
跑两次不重复插（第二次各表 inserted 为 0）。
|
||
|
||
- schema 来自 cloud.db.POSTGRES_SCHEMA / init_postgres_schema，
|
||
本模块不另发明表结构；迁移前先建表（若目标库已有表，
|
||
CREATE TABLE IF NOT EXISTS 是 no-op）。
|
||
- kb_fts（SQLite FTS5 虚拟表）不迁移：Postgres 没有 FTS5，
|
||
KBStore 在 PG 上走 LIKE 分支，kb_docs 的 content 已足够。
|
||
- sqlite_sequence（AUTOINCREMENT 内部表）不迁移。
|
||
- 不打印、不返回任何密码或密钥：函数只返回 {表: inserted}，
|
||
CLI 只打印表名与行数。
|
||
- Postgres 连接参数只走 CLOUD_DB_* 环境变量（见 cloud.db），
|
||
本模块不读、不拼连接串、不接受命令行密码参数。
|
||
"""
|
||
|
||
import os
|
||
import sqlite3
|
||
|
||
from cloud.db import (
|
||
POSTGRES_SCHEMA,
|
||
PostgresDatabase,
|
||
init_postgres_schema,
|
||
)
|
||
|
||
# 迁移顺序：无外键约束，顺序只影响日志可读性；用户表在前，
|
||
# 审计/成本等追加表在后。kb_fts 与 sqlite_sequence 明确排除。
|
||
MIGRATION_TABLES = (
|
||
"users",
|
||
"projects",
|
||
"project_members",
|
||
"tasks",
|
||
"audit_log",
|
||
"summaries",
|
||
"kb_docs",
|
||
"cost_records",
|
||
"oauth_states",
|
||
"slack_users",
|
||
"push_subscriptions",
|
||
)
|
||
|
||
_SKIP_TABLES = frozenset({"kb_fts", "sqlite_sequence"})
|
||
|
||
|
||
def migrate_sqlite_to_postgres(sqlite_path, pg_conn=None, tables=None):
|
||
"""迁移并返回 {表名: 本次插入行数}。
|
||
|
||
pg_conn 为 None 时用 PostgresDatabase().connect() 新建连接
|
||
（驱动缺失时抛 PostgresDriverMissingError，绝不回落）；
|
||
传入连接时调用方负责关闭，本函数不关闭它。tables 为 None
|
||
时迁 MIGRATION_TABLES 全量；显式传表名时只迁其中在
|
||
SQLite 里存在的表，不存在的表在结果里记 0（不报错，
|
||
因为老库可能还没建出某张表，例如未用过的 push 表）。
|
||
"""
|
||
wanted = tuple(tables) if tables is not None else MIGRATION_TABLES
|
||
src = sqlite3.connect(str(sqlite_path))
|
||
src.row_factory = sqlite3.Row
|
||
own_conn = pg_conn is None
|
||
if own_conn:
|
||
pg_conn = PostgresDatabase().connect()
|
||
try:
|
||
init_postgres_schema(pg_conn)
|
||
result = {}
|
||
for table in wanted:
|
||
if table in _SKIP_TABLES:
|
||
result[table] = 0
|
||
continue
|
||
try:
|
||
rows = src.execute(
|
||
'SELECT * FROM "%s"' % table).fetchall()
|
||
except sqlite3.OperationalError:
|
||
# 老库里没有这张表：记 0，不中断其余表。
|
||
result[table] = 0
|
||
continue
|
||
if not rows:
|
||
result[table] = 0
|
||
continue
|
||
columns = [d[0] for d in src.execute(
|
||
'SELECT * FROM "%s" LIMIT 0' % table).description]
|
||
placeholders = ", ".join("?" for _ in columns)
|
||
col_list = ", ".join('"%s"' % c for c in columns)
|
||
inserted = 0
|
||
for row in rows:
|
||
cur = pg_conn.execute(
|
||
'INSERT INTO "%s" (%s) VALUES (%s)'
|
||
" ON CONFLICT DO NOTHING"
|
||
% (table, col_list, placeholders),
|
||
tuple(row[c] for c in columns))
|
||
if cur.rowcount:
|
||
inserted += cur.rowcount
|
||
# 身份列的序列不会因显式插 id 自动前进：迁完把序列
|
||
# 拨到当前最大 id，否则后续省略 id 的 INSERT 会撞主键。
|
||
if table in ("audit_log", "summaries", "kb_docs",
|
||
"cost_records") and "id" in columns:
|
||
try:
|
||
pg_conn.execute(
|
||
"SELECT setval(pg_get_serial_sequence(?, ?),"
|
||
" COALESCE((SELECT MAX(id) FROM \"%s\"), 1), true)"
|
||
% table,
|
||
(table, "id"))
|
||
except Exception:
|
||
pass
|
||
pg_conn.commit()
|
||
result[table] = inserted
|
||
return result
|
||
finally:
|
||
src.close()
|
||
if own_conn:
|
||
try:
|
||
pg_conn.close()
|
||
except Exception:
|
||
pass
|
||
|
||
|
||
__all__ = [
|
||
"MIGRATION_TABLES",
|
||
"migrate_sqlite_to_postgres",
|
||
]
|