fix(api): bound bootstrap lock during startup
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Failing after 34s
CD Pipeline / post-deploy-checks (push) Has been cancelled
CD Pipeline / build-and-deploy (push) Has been skipped
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Failing after 34s
CD Pipeline / post-deploy-checks (push) Has been cancelled
CD Pipeline / build-and-deploy (push) Has been skipped
This commit is contained in:
@@ -13,6 +13,7 @@ Features:
|
||||
- 所有 Episodic Memory 必須使用 PostgreSQL
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from collections.abc import AsyncGenerator
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
@@ -183,6 +184,9 @@ async def get_db_context(project_id: str | None = None) -> AsyncGenerator[AsyncS
|
||||
# =============================================================================
|
||||
|
||||
_DB_BOOTSTRAP_LOCK_NAME = "awoooi:init_db:ddl"
|
||||
_DB_BOOTSTRAP_LOCK_WAIT_SECONDS = 45.0
|
||||
_DB_BOOTSTRAP_LOCK_POLL_SECONDS = 2.0
|
||||
_DB_BOOTSTRAP_DDL_WAIT_SECONDS = 120.0
|
||||
|
||||
|
||||
async def init_db() -> None:
|
||||
@@ -196,13 +200,27 @@ async def init_db() -> None:
|
||||
async with engine.connect() as lock_conn:
|
||||
# 2026-05-24 ogt + Codex: 兩個 API replica 同時啟動時,PostgreSQL 會在
|
||||
# ALTER TABLE ... IF NOT EXISTS 上互相等待並 deadlock。整段 bootstrap
|
||||
# DDL 必須序列化,避免 rollout 因一個 pod CrashLoop 變成 1/2 ready。
|
||||
await lock_conn.execute(
|
||||
text("SELECT pg_advisory_lock(hashtext(:lock_name))"),
|
||||
{"lock_name": _DB_BOOTSTRAP_LOCK_NAME},
|
||||
)
|
||||
# DDL 必須序列化。
|
||||
# 2026-06-30 Codex: production rollout 觀察到新 API pod 卡在 init_db()
|
||||
# 超過 startup probe 預算。bootstrap DDL 是冪等補欄流程,不能再用
|
||||
# 無界 pg_advisory_lock 把新版 API route 擋成永久舊版;改為有界等待,
|
||||
# 逾時則 fail-visible skip,讓 serving path 先恢復,再由 verifier 回報。
|
||||
lock_acquired = await _try_acquire_bootstrap_lock(lock_conn)
|
||||
if not lock_acquired:
|
||||
return
|
||||
|
||||
try:
|
||||
await _run_init_db_ddl(engine)
|
||||
try:
|
||||
await asyncio.wait_for(
|
||||
_run_init_db_ddl(engine),
|
||||
timeout=_DB_BOOTSTRAP_DDL_WAIT_SECONDS,
|
||||
)
|
||||
except TimeoutError:
|
||||
logger.warning(
|
||||
"database_bootstrap_ddl_timeout_skipped",
|
||||
timeout_seconds=_DB_BOOTSTRAP_DDL_WAIT_SECONDS,
|
||||
lock_name=_DB_BOOTSTRAP_LOCK_NAME,
|
||||
)
|
||||
finally:
|
||||
await lock_conn.execute(
|
||||
text("SELECT pg_advisory_unlock(hashtext(:lock_name))"),
|
||||
@@ -210,6 +228,34 @@ async def init_db() -> None:
|
||||
)
|
||||
|
||||
|
||||
async def _try_acquire_bootstrap_lock(lock_conn: object) -> bool:
|
||||
loop = asyncio.get_running_loop()
|
||||
deadline = loop.time() + _DB_BOOTSTRAP_LOCK_WAIT_SECONDS
|
||||
|
||||
while True:
|
||||
result = await lock_conn.execute(
|
||||
text("SELECT pg_try_advisory_lock(hashtext(:lock_name))"),
|
||||
{"lock_name": _DB_BOOTSTRAP_LOCK_NAME},
|
||||
)
|
||||
if bool(result.scalar()):
|
||||
logger.info(
|
||||
"database_bootstrap_lock_acquired",
|
||||
lock_name=_DB_BOOTSTRAP_LOCK_NAME,
|
||||
)
|
||||
return True
|
||||
|
||||
remaining = deadline - loop.time()
|
||||
if remaining <= 0:
|
||||
logger.warning(
|
||||
"database_bootstrap_lock_timeout_skipped",
|
||||
timeout_seconds=_DB_BOOTSTRAP_LOCK_WAIT_SECONDS,
|
||||
lock_name=_DB_BOOTSTRAP_LOCK_NAME,
|
||||
)
|
||||
return False
|
||||
|
||||
await asyncio.sleep(min(_DB_BOOTSTRAP_LOCK_POLL_SECONDS, remaining))
|
||||
|
||||
|
||||
async def _run_init_db_ddl(engine: AsyncEngine) -> None:
|
||||
"""
|
||||
Run idempotent DB bootstrap DDL while caller holds the bootstrap advisory lock.
|
||||
|
||||
Reference in New Issue
Block a user