fix(worker): dedicated Redis pool with unlimited timeout for XREADGROUP

Root cause: Worker shared Redis pool with API (socket_timeout=5s),
but XREADGROUP blocks for 5s causing timeout errors every cycle.

Fix:
- Add init_worker_redis_pool() with socket_timeout=None
- Worker now uses get_worker_redis() for XREADGROUP operations
- API continues using get_redis() with short timeout

Also destroyed 50 zombie consumers via:
  XGROUP DESTROY stream:awoooi_signals awoooi_workers

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
OG T
2026-03-23 09:42:11 +08:00
parent 80d0ef4a8f
commit 9f353343c9
2 changed files with 85 additions and 6 deletions

View File

@@ -27,11 +27,17 @@ logger = structlog.get_logger(__name__)
# =============================================================================
# Connection Pool
# Connection Pool (API 用 - 短超時)
# =============================================================================
_redis_pool: redis.Redis | None = None
# =============================================================================
# Worker 專屬連線池 (長超時,用於 XREADGROUP 阻塞操作)
# =============================================================================
_worker_redis_pool: redis.Redis | None = None
async def init_redis_pool() -> redis.Redis:
"""
@@ -83,7 +89,7 @@ async def close_redis_pool() -> None:
def get_redis() -> redis.Redis:
"""
取得 Redis 連線
取得 Redis 連線 (API 用,短超時)
Raises:
RuntimeError: 若連線池未初始化
@@ -93,6 +99,73 @@ def get_redis() -> redis.Redis:
return _redis_pool
# =============================================================================
# Worker 專屬連線池 (長超時)
# =============================================================================
async def init_worker_redis_pool() -> redis.Redis:
"""
初始化 Worker 專屬 Redis 連線池
統帥鐵律 2026-03-23:
- Worker 使用 XREADGROUP 阻塞操作,需要長超時
- 絕對禁止與 API 共用短超時連線池
- socket_timeout=None 表示無限等待
"""
global _worker_redis_pool
if _worker_redis_pool is not None:
return _worker_redis_pool
_worker_redis_pool = redis.from_url(
settings.REDIS_URL,
encoding="utf-8",
decode_responses=True,
max_connections=5, # Worker 不需要太多連線
socket_timeout=None, # 無限等待 (XREADGROUP 阻塞操作)
socket_connect_timeout=10.0, # 連線超時仍需設定
)
# 測試連線
try:
await _worker_redis_pool.ping()
logger.info(
"worker_redis_pool_initialized",
url=settings.REDIS_URL.split("@")[-1],
socket_timeout="None (unlimited)",
)
except redis.ConnectionError as e:
logger.error("worker_redis_connection_failed", error=str(e))
raise
return _worker_redis_pool
async def close_worker_redis_pool() -> None:
"""
關閉 Worker 專屬 Redis 連線池
"""
global _worker_redis_pool
if _worker_redis_pool is not None:
await _worker_redis_pool.close()
_worker_redis_pool = None
logger.info("worker_redis_pool_closed")
def get_worker_redis() -> redis.Redis:
"""
取得 Worker 專屬 Redis 連線 (長超時,用於 XREADGROUP)
Raises:
RuntimeError: 若連線池未初始化
"""
if _worker_redis_pool is None:
raise RuntimeError("Worker Redis pool not initialized. Call init_worker_redis_pool() first.")
return _worker_redis_pool
# =============================================================================
# Distributed Lock (分散式鎖)
# =============================================================================