fix(api): bound bootstrap lock during startup
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Failing after 34s
CD Pipeline / post-deploy-checks (push) Has been cancelled
CD Pipeline / build-and-deploy (push) Has been skipped

This commit is contained in:
Your Name
2026-06-30 08:19:40 +08:00
parent 15b68a0767
commit 5f99459d1b
3 changed files with 134 additions and 13 deletions

View File

@@ -13,6 +13,7 @@ Features:
- 所有 Episodic Memory 必須使用 PostgreSQL
"""
import asyncio
from collections.abc import AsyncGenerator
from contextlib import asynccontextmanager
@@ -183,6 +184,9 @@ async def get_db_context(project_id: str | None = None) -> AsyncGenerator[AsyncS
# =============================================================================
_DB_BOOTSTRAP_LOCK_NAME = "awoooi:init_db:ddl"
_DB_BOOTSTRAP_LOCK_WAIT_SECONDS = 45.0
_DB_BOOTSTRAP_LOCK_POLL_SECONDS = 2.0
_DB_BOOTSTRAP_DDL_WAIT_SECONDS = 120.0
async def init_db() -> None:
@@ -196,13 +200,27 @@ async def init_db() -> None:
async with engine.connect() as lock_conn:
# 2026-05-24 ogt + Codex: 兩個 API replica 同時啟動時PostgreSQL 會在
# ALTER TABLE ... IF NOT EXISTS 上互相等待並 deadlock。整段 bootstrap
# DDL 必須序列化,避免 rollout 因一個 pod CrashLoop 變成 1/2 ready
await lock_conn.execute(
text("SELECT pg_advisory_lock(hashtext(:lock_name))"),
{"lock_name": _DB_BOOTSTRAP_LOCK_NAME},
)
# DDL 必須序列化。
# 2026-06-30 Codex: production rollout 觀察到新 API pod 卡在 init_db()
# 超過 startup probe 預算。bootstrap DDL 是冪等補欄流程,不能再用
# 無界 pg_advisory_lock 把新版 API route 擋成永久舊版;改為有界等待,
# 逾時則 fail-visible skip讓 serving path 先恢復,再由 verifier 回報。
lock_acquired = await _try_acquire_bootstrap_lock(lock_conn)
if not lock_acquired:
return
try:
await _run_init_db_ddl(engine)
try:
await asyncio.wait_for(
_run_init_db_ddl(engine),
timeout=_DB_BOOTSTRAP_DDL_WAIT_SECONDS,
)
except TimeoutError:
logger.warning(
"database_bootstrap_ddl_timeout_skipped",
timeout_seconds=_DB_BOOTSTRAP_DDL_WAIT_SECONDS,
lock_name=_DB_BOOTSTRAP_LOCK_NAME,
)
finally:
await lock_conn.execute(
text("SELECT pg_advisory_unlock(hashtext(:lock_name))"),
@@ -210,6 +228,34 @@ async def init_db() -> None:
)
async def _try_acquire_bootstrap_lock(lock_conn: object) -> bool:
loop = asyncio.get_running_loop()
deadline = loop.time() + _DB_BOOTSTRAP_LOCK_WAIT_SECONDS
while True:
result = await lock_conn.execute(
text("SELECT pg_try_advisory_lock(hashtext(:lock_name))"),
{"lock_name": _DB_BOOTSTRAP_LOCK_NAME},
)
if bool(result.scalar()):
logger.info(
"database_bootstrap_lock_acquired",
lock_name=_DB_BOOTSTRAP_LOCK_NAME,
)
return True
remaining = deadline - loop.time()
if remaining <= 0:
logger.warning(
"database_bootstrap_lock_timeout_skipped",
timeout_seconds=_DB_BOOTSTRAP_LOCK_WAIT_SECONDS,
lock_name=_DB_BOOTSTRAP_LOCK_NAME,
)
return False
await asyncio.sleep(min(_DB_BOOTSTRAP_LOCK_POLL_SECONDS, remaining))
async def _run_init_db_ddl(engine: AsyncEngine) -> None:
"""
Run idempotent DB bootstrap DDL while caller holds the bootstrap advisory lock.