Startup race: concurrent init_db (cloud-server + worker) ran CREATE TABLE / CREATE INDEX IF NOT EXISTS against a DB where the object did not exist yet; the check-then-create is not atomic, so both sessions created it and one died with UniqueViolation on pg_type_typname_nsp_index (tables) or pg_class_relname_nsp_index (indexes). Hit on the37ba687and7983567deploys (7983567added idx_tasks_parent); systemd restart healed it every time. Fix (advisory lock, not catch-duplicate_object): init_postgres_schema takes pg_advisory_xact_lock (key 20261108, released at its commit), and server.init_db takes the session-level lock for the whole PG branch so the _ensure_* objects outside POSTGRES_SCHEMA (task_continue_messages, user_gateway_keys, task_attempts, task_audit_loop_state) are covered too. Same idiom as the existing RATE_LIMIT_SWEEP_ADVISORY_KEY sweep lock; key is distinct. Tests: two new real-PG regression tests (skipped without CLOUD_TEST_PG_DSN), 4 processes racing on an empty schema, for init_postgres_schema and for full init_db; both fail on the unfixed code and pass with the fix.
This commit is contained in:
1 parent
7983567491
commit
8b2686548c
4 files changed
+238
-16
No files matched your search
+28
-16
@@ -88,7 +88,9 @@ from cloud.timeouts import validate_server_timeout as _validate_timeout
|
||||
from cloud.auth import audit_on_conn as _audit_on_conn
|
||||
from cloud.db import connect as _db_connect
|
||||
from cloud.db import create_database as _create_database
|
||||
from cloud.db import acquire_schema_init_lock as _acquire_schema_init_lock
|
||||
from cloud.db import init_postgres_schema as _init_postgres_schema
|
||||
from cloud.db import release_schema_init_lock as _release_schema_init_lock
|
||||
from cloud.db import PostgresIntegrityError as _PgIntegrityError
|
||||
from cloud.db import PostgresOperationalError as _PgOperationalError
|
||||
from cloud.goal_command import parse_goal_command as _parse_goal_command
|
||||
@@ -1667,22 +1669,32 @@ def init_db(db_path):
|
||||
conn = _connect(db_path)
|
||||
try:
|
||||
if _is_postgres_backend():
|
||||
_init_postgres_schema(conn)
|
||||
migrate_tasks_table(conn)
|
||||
migrate_projects_table(conn)
|
||||
_ensure_followups_table(conn)
|
||||
_ensure_continue_messages_table(conn)
|
||||
_ensure_environments_table(conn)
|
||||
_ensure_quotas_table(conn)
|
||||
_backfill_default_quotas(conn)
|
||||
_ensure_gateway_keys(conn)
|
||||
_ensure_uploads_tables(conn)
|
||||
_ensure_task_checkpoints_table(conn)
|
||||
_ensure_task_steps_table(conn)
|
||||
_ensure_attempts(conn)
|
||||
_ensure_audit_loop(conn)
|
||||
conn.commit()
|
||||
return
|
||||
# 会话级 advisory 锁覆盖整个 PG 初始化:_init_postgres_schema
|
||||
# 内还有事务级锁,但其后的 _ensure_* 建的对象不全在
|
||||
# POSTGRES_SCHEMA 里(task_continue_messages /
|
||||
# user_gateway_keys / task_attempts / task_audit_loop_state),
|
||||
# 只锁前者仍会在后者上撞 UniqueViolation;锁在最终 commit
|
||||
# 后才放。连接在外层 finally 里关闭,锁至迟随连接释放。
|
||||
_acquire_schema_init_lock(conn)
|
||||
try:
|
||||
_init_postgres_schema(conn)
|
||||
migrate_tasks_table(conn)
|
||||
migrate_projects_table(conn)
|
||||
_ensure_followups_table(conn)
|
||||
_ensure_continue_messages_table(conn)
|
||||
_ensure_environments_table(conn)
|
||||
_ensure_quotas_table(conn)
|
||||
_backfill_default_quotas(conn)
|
||||
_ensure_gateway_keys(conn)
|
||||
_ensure_uploads_tables(conn)
|
||||
_ensure_task_checkpoints_table(conn)
|
||||
_ensure_task_steps_table(conn)
|
||||
_ensure_attempts(conn)
|
||||
_ensure_audit_loop(conn)
|
||||
conn.commit()
|
||||
return
|
||||
finally:
|
||||
_release_schema_init_lock(conn)
|
||||
conn.execute(
|
||||
"CREATE TABLE IF NOT EXISTS projects("
|
||||
"id TEXT PRIMARY KEY, "
|
||||
|
||||
Reference in new issue
Block a user