host
7091a2d0d2
- CODEX_GLOBAL_ADMINS replaces maintainer-derived global admin; admin endpoints (quotas, sweep, backup, push dispatch) fail-closed 403 when unset, with audit on allow and deny. - continue/followups/messages/retry check both caller and original creator quotas (USD + unpriced token cap), 402 on either overrun. - Quota-less users now get the #86 default quota instead of unlimited; init_db backfills and OAuth create/bind/existing-login branches INSERT OR IGNORE a default row; explicit NULL row stays unlimited. user_quotas added to migration tables. - /fix-review: untrusted authors rejected before dedup claim, no task; trusted tasks force executor=docker, created_by=project owner, owner quota gate, prompt is a fixed template plus a 2000-char excerpt. - GitHub App setup state only issued by global admins, state rows bind issued_by (ALTER-compatible for old DBs), callback rejects issuers no longer in the admin list without consuming the state. - Cancel: popen poll checks DB status every second, Docker path runs a cancel watchdog that kills by container name, write-back is CAS-guarded so late results never overwrite a new status. - SHELL_ENV_EXCLUDE adds *_TOKEN, *_AUTH*, *_CREDENTIAL*, *API_KEY*; comment states the residual risk: same-UID /proc environ remains readable, true isolation needs UID separation or hidepid. - Tests: new cloud/tests/test_p0_security.py (17 cases); old tests updated to the new semantics (admin env, default quota, trusted fix-review author) without weakening any gate.
125 lines
4.6 KiB
Python
125 lines
4.6 KiB
Python
"""SQLite -> Postgres 迁移（控制面状态库）。
|
||
|
||
只做一件事：把一个 SQLite 库文件里的行原样插进 Postgres，
|
||
表名与列名不改。幂等：Postgres 侧用 ``ON CONFLICT DO NOTHING``，
|
||
跑两次不重复插（第二次各表 inserted 为 0）。
|
||
|
||
- schema 来自 cloud.db.POSTGRES_SCHEMA / init_postgres_schema，
|
||
本模块不另发明表结构；迁移前先建表（若目标库已有表，
|
||
CREATE TABLE IF NOT EXISTS 是 no-op）。
|
||
- kb_fts（SQLite FTS5 虚拟表）不迁移：Postgres 没有 FTS5，
|
||
KBStore 在 PG 上走 LIKE 分支，kb_docs 的 content 已足够。
|
||
- sqlite_sequence（AUTOINCREMENT 内部表）不迁移。
|
||
- 不打印、不返回任何密码或密钥：函数只返回 {表: inserted}，
|
||
CLI 只打印表名与行数。
|
||
- Postgres 连接参数只走 CLOUD_DB_* 环境变量（见 cloud.db），
|
||
本模块不读、不拼连接串、不接受命令行密码参数。
|
||
"""
|
||
|
||
import os
|
||
import sqlite3
|
||
|
||
from cloud.db import (
|
||
POSTGRES_SCHEMA,
|
||
PostgresDatabase,
|
||
init_postgres_schema,
|
||
)
|
||
|
||
# 迁移顺序：无外键约束，顺序只影响日志可读性；用户表在前，
|
||
# 审计/成本等追加表在后。kb_fts 与 sqlite_sequence 明确排除。
|
||
MIGRATION_TABLES = (
|
||
"users",
|
||
"projects",
|
||
"project_members",
|
||
"tasks",
|
||
"audit_log",
|
||
"summaries",
|
||
"kb_docs",
|
||
"cost_records",
|
||
"oauth_states",
|
||
"slack_users",
|
||
"push_subscriptions",
|
||
# P0 安全批：配额行必须随库迁移，否则迁后用户变回无行
|
||
# （本批后无行按默认额度处理，不再是无限额，但显式
|
||
# 管理员额度仍需原样迁过去，不能丢）。
|
||
"user_quotas",
|
||
)
|
||
|
||
_SKIP_TABLES = frozenset({"kb_fts", "sqlite_sequence"})
|
||
|
||
|
||
def migrate_sqlite_to_postgres(sqlite_path, pg_conn=None, tables=None):
|
||
"""迁移并返回 {表名: 本次插入行数}。
|
||
|
||
pg_conn 为 None 时用 PostgresDatabase().connect() 新建连接
|
||
（驱动缺失时抛 PostgresDriverMissingError，绝不回落）；
|
||
传入连接时调用方负责关闭，本函数不关闭它。tables 为 None
|
||
时迁 MIGRATION_TABLES 全量；显式传表名时只迁其中在
|
||
SQLite 里存在的表，不存在的表在结果里记 0（不报错，
|
||
因为老库可能还没建出某张表，例如未用过的 push 表）。
|
||
"""
|
||
wanted = tuple(tables) if tables is not None else MIGRATION_TABLES
|
||
src = sqlite3.connect(str(sqlite_path))
|
||
src.row_factory = sqlite3.Row
|
||
own_conn = pg_conn is None
|
||
if own_conn:
|
||
pg_conn = PostgresDatabase().connect()
|
||
try:
|
||
init_postgres_schema(pg_conn)
|
||
result = {}
|
||
for table in wanted:
|
||
if table in _SKIP_TABLES:
|
||
result[table] = 0
|
||
continue
|
||
try:
|
||
rows = src.execute(
|
||
'SELECT * FROM "%s"' % table).fetchall()
|
||
except sqlite3.OperationalError:
|
||
# 老库里没有这张表：记 0，不中断其余表。
|
||
result[table] = 0
|
||
continue
|
||
if not rows:
|
||
result[table] = 0
|
||
continue
|
||
columns = [d[0] for d in src.execute(
|
||
'SELECT * FROM "%s" LIMIT 0' % table).description]
|
||
placeholders = ", ".join("?" for _ in columns)
|
||
col_list = ", ".join('"%s"' % c for c in columns)
|
||
inserted = 0
|
||
for row in rows:
|
||
cur = pg_conn.execute(
|
||
'INSERT INTO "%s" (%s) VALUES (%s)'
|
||
" ON CONFLICT DO NOTHING"
|
||
% (table, col_list, placeholders),
|
||
tuple(row[c] for c in columns))
|
||
if cur.rowcount:
|
||
inserted += cur.rowcount
|
||
# 身份列的序列不会因显式插 id 自动前进：迁完把序列
|
||
# 拨到当前最大 id，否则后续省略 id 的 INSERT 会撞主键。
|
||
if table in ("audit_log", "summaries", "kb_docs",
|
||
"cost_records") and "id" in columns:
|
||
try:
|
||
pg_conn.execute(
|
||
"SELECT setval(pg_get_serial_sequence(?, ?),"
|
||
" COALESCE((SELECT MAX(id) FROM \"%s\"), 1), true)"
|
||
% table,
|
||
(table, "id"))
|
||
except Exception:
|
||
pass
|
||
pg_conn.commit()
|
||
result[table] = inserted
|
||
return result
|
||
finally:
|
||
src.close()
|
||
if own_conn:
|
||
try:
|
||
pg_conn.close()
|
||
except Exception:
|
||
pass
|
||
|
||
|
||
__all__ = [
|
||
"MIGRATION_TABLES",
|
||
"migrate_sqlite_to_postgres",
|
||
]
|