fix(api): cap db pool during prod rollout

apps/api/src/db/base.py

@@ -19,7 +19,9 @@ from contextlib import asynccontextmanager
 
 from fastapi import HTTPException
 from sqlalchemy import text
+from sqlalchemy.exc import DBAPIError, TimeoutError as SQLAlchemyTimeoutError
 from sqlalchemy.ext.asyncio import (
+    AsyncConnection,
     AsyncEngine,
     AsyncSession,
     async_sessionmaker,
@@ -87,8 +89,9 @@ def get_engine() -> AsyncEngine:
         _engine = create_async_engine(
             database_url,
             echo=settings.DEBUG,
-            pool_size=10,
-            max_overflow=20,
+            pool_size=settings.DATABASE_POOL_SIZE,
+            max_overflow=settings.DATABASE_MAX_OVERFLOW,
+            pool_timeout=settings.DATABASE_POOL_TIMEOUT_SECONDS,
             pool_pre_ping=True,
         )
     return _engine
@@ -189,6 +192,30 @@ _DB_BOOTSTRAP_LOCK_POLL_SECONDS = 2.0
 _DB_BOOTSTRAP_DDL_WAIT_SECONDS = 120.0
 
 
+def _is_database_connection_budget_error(exc: BaseException) -> bool:
+    """Return True for PostgreSQL role/pool exhaustion without hiding other DB errors."""
+    seen: set[int] = set()
+    current: BaseException | None = exc
+    budget_markers = (
+        "too many connections",
+        "remaining connection slots are reserved",
+        "connection limit",
+        "queuepool limit",
+    )
+
+    while current is not None and id(current) not in seen:
+        seen.add(id(current))
+        message = f"{type(current).__name__}: {current}".lower()
+        if any(marker in message for marker in budget_markers):
+            return True
+        current = (
+            getattr(current, "orig", None)
+            or getattr(current, "__cause__", None)
+            or getattr(current, "__context__", None)
+        )
+    return False
+
+
 async def init_db() -> None:
     """
     Initialize database tables
@@ -197,35 +224,50 @@ async def init_db() -> None:
     """
     engine = get_engine()
 
-    async with engine.connect() as lock_conn:
-        # 2026-05-24 ogt + Codex: 兩個 API replica 同時啟動時，PostgreSQL 會在
-        # ALTER TABLE ... IF NOT EXISTS 上互相等待並 deadlock。整段 bootstrap
-        # DDL 必須序列化。
-        # 2026-06-30 Codex: production rollout 觀察到新 API pod 卡在 init_db()
-        # 超過 startup probe 預算。bootstrap DDL 是冪等補欄流程，不能再用
-        # 無界 pg_advisory_lock 把新版 API route 擋成永久舊版；改為有界等待，
-        # 逾時則 fail-visible skip，讓 serving path 先恢復，再由 verifier 回報。
-        lock_acquired = await _try_acquire_bootstrap_lock(lock_conn)
-        if not lock_acquired:
-            return
+    try:
+        async with engine.connect() as lock_conn:
+            # 2026-05-24 ogt + Codex: 兩個 API replica 同時啟動時，PostgreSQL 會在
+            # ALTER TABLE ... IF NOT EXISTS 上互相等待並 deadlock。整段 bootstrap
+            # DDL 必須序列化。
+            # 2026-06-30 Codex: production rollout 觀察到新 API pod 卡在 init_db()
+            # 超過 startup probe 預算。bootstrap DDL 是冪等補欄流程，不能再用
+            # 無界 pg_advisory_lock 把新版 API route 擋成永久舊版；改為有界等待，
+            # 逾時則 fail-visible skip，讓 serving path 先恢復，再由 verifier 回報。
+            # 2026-07-01 Codex: production DB role has CONNECTION LIMIT 2. Keep
+            # bootstrap on the lock connection so pool_size=1 can still start.
+            lock_acquired = await _try_acquire_bootstrap_lock(lock_conn)
+            if not lock_acquired:
+                return
 
-        try:
             try:
-                await asyncio.wait_for(
-                    _run_init_db_ddl(engine),
-                    timeout=_DB_BOOTSTRAP_DDL_WAIT_SECONDS,
+                try:
+                    await asyncio.wait_for(
+                        _run_init_db_ddl(lock_conn),
+                        timeout=_DB_BOOTSTRAP_DDL_WAIT_SECONDS,
+                    )
+                except TimeoutError:
+                    logger.warning(
+                        "database_bootstrap_ddl_timeout_skipped",
+                        timeout_seconds=_DB_BOOTSTRAP_DDL_WAIT_SECONDS,
+                        lock_name=_DB_BOOTSTRAP_LOCK_NAME,
+                    )
+            finally:
+                await lock_conn.execute(
+                    text("SELECT pg_advisory_unlock(hashtext(:lock_name))"),
+                    {"lock_name": _DB_BOOTSTRAP_LOCK_NAME},
                 )
-            except TimeoutError:
-                logger.warning(
-                    "database_bootstrap_ddl_timeout_skipped",
-                    timeout_seconds=_DB_BOOTSTRAP_DDL_WAIT_SECONDS,
-                    lock_name=_DB_BOOTSTRAP_LOCK_NAME,
-                )
-        finally:
-            await lock_conn.execute(
-                text("SELECT pg_advisory_unlock(hashtext(:lock_name))"),
-                {"lock_name": _DB_BOOTSTRAP_LOCK_NAME},
+                await lock_conn.commit()
+    except (DBAPIError, SQLAlchemyTimeoutError, OSError, RuntimeError) as exc:
+        if _is_database_connection_budget_error(exc):
+            logger.warning(
+                "database_bootstrap_connection_budget_exhausted_skipped",
+                error_type=type(exc).__name__,
+                lock_name=_DB_BOOTSTRAP_LOCK_NAME,
+                pool_size=settings.DATABASE_POOL_SIZE,
+                max_overflow=settings.DATABASE_MAX_OVERFLOW,
             )
+            return
+        raise
 
 
 async def _try_acquire_bootstrap_lock(lock_conn: object) -> bool:
@@ -256,7 +298,7 @@ async def _try_acquire_bootstrap_lock(lock_conn: object) -> bool:
         await asyncio.sleep(min(_DB_BOOTSTRAP_LOCK_POLL_SECONDS, remaining))
 
 
-async def _run_init_db_ddl(engine: AsyncEngine) -> None:
+async def _run_init_db_ddl(conn: AsyncConnection) -> None:
     """
     Run idempotent DB bootstrap DDL while caller holds the bootstrap advisory lock.
     """
@@ -267,15 +309,15 @@ async def _run_init_db_ddl(engine: AsyncEngine) -> None:
     #       → table + index 全消失 → 下次重啟同樣問題 → 無限 CrashLoop
     # 修法：每個 table 獨立 transaction；先 DROP INDEX IF EXISTS 清殘留孤兒 index；
     #       捕捉 "already exists" 讓並行 pod 優雅跳過
-    async with engine.connect() as probe_conn:
-        existing = set(await probe_conn.run_sync(
-            lambda c: __import__('sqlalchemy', fromlist=['inspect']).inspect(c).get_table_names()
-        ))
+    existing = set(await conn.run_sync(
+        lambda c: __import__('sqlalchemy', fromlist=['inspect']).inspect(c).get_table_names()
+    ))
+    await conn.commit()
 
     for table in Base.metadata.sorted_tables:
         if table.name not in existing:
             try:
-                async with engine.begin() as conn:
+                async with conn.begin():
                     # 先清殘留孤兒 index（前次 CrashLoop 留下的部分狀態）
                     for index in table.indexes:
                         await conn.execute(text(f'DROP INDEX IF EXISTS "{index.name}"'))
@@ -286,7 +328,7 @@ async def _run_init_db_ddl(engine: AsyncEngine) -> None:
                 else:
                     raise
 
-    async with engine.begin() as conn:
+    async with conn.begin():
         # 2026-04-02 Claude Code: 確保 risklevel enum 包含 'high' 值
         # Phase 23 新增，避免舊 DB 缺少此值導致 InvalidTextRepresentation
         await conn.execute(