fix(worker): dedicated Redis pool with unlimited timeout for XREADGROUP

Root cause: Worker shared Redis pool with API (socket_timeout=5s),
but XREADGROUP blocks for 5s causing timeout errors every cycle.

Fix:
- Add init_worker_redis_pool() with socket_timeout=None
- Worker now uses get_worker_redis() for XREADGROUP operations
- API continues using get_redis() with short timeout

Also destroyed 50 zombie consumers via:
  XGROUP DESTROY stream:awoooi_signals awoooi_workers

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
OG T
2026-03-23 09:42:11 +08:00
parent 80d0ef4a8f
commit 9f353343c9
2 changed files with 85 additions and 6 deletions

View File

@@ -26,7 +26,7 @@ from typing import Any
import structlog
from src.core.redis_client import get_redis
from src.core.redis_client import get_redis, get_worker_redis
from src.services.incident_engine import get_incident_engine
logger = structlog.get_logger(__name__)
@@ -149,8 +149,12 @@ class SignalWorker:
主消費循環
XREADGROUP 阻塞等待新訊息,處理後 XACK。
統帥鐵律 2026-03-23:
- 使用 Worker 專屬 Redis 連線 (無超時限制)
- 絕對禁止使用 API 共用的短超時連線
"""
redis_client = get_redis()
redis_client = get_worker_redis() # Worker 專屬長連線
while self._running:
try:
@@ -321,7 +325,7 @@ async def _main() -> None:
# Initialize settings first (loads env vars)
from src.core.config import settings # noqa: F401
from src.core.redis_client import init_redis_pool, close_redis_pool
from src.core.redis_client import init_redis_pool, close_redis_pool, init_worker_redis_pool, close_worker_redis_pool
logger.info(
"signal_worker_standalone_starting",
@@ -329,8 +333,9 @@ async def _main() -> None:
redis_url=settings.REDIS_URL.split("@")[-1] if settings.REDIS_URL else "N/A",
)
# Initialize Redis
# Initialize Redis (API pool + Worker 專屬長連線池)
await init_redis_pool()
await init_worker_redis_pool() # Worker 專屬,無超時限制
# Write health files for K8s probes
await _write_health_files()
@@ -354,6 +359,7 @@ async def _main() -> None:
# Graceful shutdown
logger.info("signal_worker_shutting_down")
await worker.stop()
await close_worker_redis_pool() # 關閉 Worker 專屬連線
await close_redis_pool()
# Remove health files