Startup race: concurrent init_db (cloud-server + worker) ran CREATE TABLE / CREATE INDEX IF NOT EXISTS against a DB where the object did not exist yet; the check-then-create is not atomic, so both sessions created it and one died with UniqueViolation on pg_type_typname_nsp_index (tables) or pg_class_relname_nsp_index (indexes). Hit on the37ba687and7983567deploys (7983567added idx_tasks_parent); systemd restart healed it every time. Fix (advisory lock, not catch-duplicate_object): init_postgres_schema takes pg_advisory_xact_lock (key 20261108, released at its commit), and server.init_db takes the session-level lock for the whole PG branch so the _ensure_* objects outside POSTGRES_SCHEMA (task_continue_messages, user_gateway_keys, task_attempts, task_audit_loop_state) are covered too. Same idiom as the existing RATE_LIMIT_SWEEP_ADVISORY_KEY sweep lock; key is distinct. Tests: two new real-PG regression tests (skipped without CLOUD_TEST_PG_DSN), 4 processes racing on an empty schema, for init_postgres_schema and for full init_db; both fail on the unfixed code and pass with the fix.
This commit is contained in:
1 parent
7983567491
commit
8b2686548c
4 files changed
+238
-16
No files matched your search
@@ -17,7 +17,9 @@
|
||||
import os
|
||||
import socket
|
||||
import sys
|
||||
import traceback
|
||||
import unittest
|
||||
from multiprocessing import Barrier, Process, Queue
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
from urllib.parse import unquote, urlparse
|
||||
@@ -386,3 +388,81 @@ class RealPostgresSchemaTestCase(unittest.TestCase):
|
||||
except Exception:
|
||||
pass
|
||||
conn.close()
|
||||
|
||||
|
||||
def _concurrent_schema_worker(barrier, queue, wid, params):
|
||||
"""并发初始化子进程:与其它进程同时跑 init_postgres_schema。"""
|
||||
try:
|
||||
host, port, dbname, user, password = params
|
||||
barrier.wait(timeout=60)
|
||||
db = PostgresDatabase(host=host, port=port, dbname=dbname,
|
||||
user=user, password=password)
|
||||
conn = db.connect()
|
||||
try:
|
||||
init_postgres_schema(conn)
|
||||
finally:
|
||||
conn.close()
|
||||
queue.put((wid, "ok", ""))
|
||||
except Exception:
|
||||
queue.put((wid, "FAIL", traceback.format_exc()))
|
||||
|
||||
|
||||
class RealPostgresConcurrentInitTestCase(unittest.TestCase):
|
||||
"""真实 PG 并发初始化:无 DSN 或不可达时 skip。
|
||||
|
||||
回归:启动竞态(37ba687 与 7983567 两次部署均撞上)——
|
||||
cloud-server 与 worker 并发 init 时,CREATE TABLE / CREATE
|
||||
INDEX 的 IF NOT EXISTS 检查与建对象不是原子的,并发建同名
|
||||
对象会撞 pg_type_typname_nsp_index(建表)或
|
||||
pg_class_relname_nsp_index(建索引)的 UniqueViolation。
|
||||
修法是 schema 初始化持 advisory 锁串行化;本用例在空库上
|
||||
4 进程同时初始化,必须全部成功、表齐全、无一例异常外泄。
|
||||
"""
|
||||
|
||||
def test_concurrent_init_postgres_schema_no_unique_violation(self):
|
||||
params = _real_pg_params()
|
||||
if params is None:
|
||||
self.skipTest("未设置可达的 CLOUD_TEST_PG_DSN,跳过真实 PG 测试")
|
||||
if not _has_real_driver():
|
||||
self.skipTest("未装 psycopg 驱动,跳过真实 PG 测试")
|
||||
host, port, dbname, user, password = params
|
||||
db = PostgresDatabase(host=host, port=port, dbname=dbname,
|
||||
user=user, password=password)
|
||||
conn = db.connect()
|
||||
try:
|
||||
conn.execute("DROP SCHEMA public CASCADE")
|
||||
conn.execute("CREATE SCHEMA public")
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
nworkers = 4
|
||||
barrier = Barrier(nworkers)
|
||||
queue = Queue()
|
||||
procs = [Process(target=_concurrent_schema_worker,
|
||||
args=(barrier, queue, i, params))
|
||||
for i in range(nworkers)]
|
||||
try:
|
||||
for proc in procs:
|
||||
proc.start()
|
||||
for proc in procs:
|
||||
proc.join(timeout=120)
|
||||
finally:
|
||||
for proc in procs:
|
||||
if proc.is_alive():
|
||||
proc.terminate()
|
||||
results = [queue.get() for _ in range(nworkers)]
|
||||
failures = [r for r in results if r[1] == "FAIL"]
|
||||
self.assertEqual([], failures,
|
||||
"并发初始化有进程失败:%s" % (failures,))
|
||||
conn = db.connect()
|
||||
try:
|
||||
rows = db.read_all(
|
||||
conn,
|
||||
"SELECT table_name FROM information_schema.tables"
|
||||
" WHERE table_schema = 'public'",
|
||||
())
|
||||
names = {r["table_name"] for r in rows}
|
||||
for table in SchemaStatementTestCase.EXPECTED_TABLES:
|
||||
self.assertIn(table, names)
|
||||
finally:
|
||||
conn.close()
|
||||
Reference in new issue
Block a user