fix(telegram): Redis Leader Election 解決多 Pod 409 Conflict
Some checks failed
CD Pipeline / build-and-deploy (push) Has been cancelled
E2E Health Check / e2e-health (push) Has been cancelled

問題: 2 個 API Pod 同時 getUpdates → 互相 409 → 兩個都失敗
根本原因: explicit env TELEGRAM_ENABLE_POLLING=false 被 kubectl patch 設入
  deployment,覆蓋 ConfigMap 的 true (feedback_k8s_env_precedence.md 違規)

修復步驟:
1. kubectl patch 移除 deployment 的 explicit env override
2. 實作 Redis Leader Election 防止多 Pod 競爭
   - 使用 SET NX EX=45 取得 Leader Lock
   - _leader_renewer(): 每 20s 續約,確保 Leader 持有 Lock
   - _leader_watcher(): 非 Leader Pod 每 30s 嘗試接管
   - 409 時主動釋放 Lock,Watcher 競爭接管

結果: 一個 Pod 正常 polling,另一個 Pod 進入 Watcher 待命模式

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
OG T
2026-04-01 11:04:10 +08:00
parent b3e30e7d84
commit e6f6734f39

View File

@@ -24,6 +24,7 @@ SOUL.md 鐵律 (4.1 Telegram 訊息壓縮原則):
import asyncio
import html
import os
from dataclasses import dataclass
from datetime import UTC, datetime
@@ -48,6 +49,13 @@ SILENCE_KEY_PREFIX = "telegram_silence:" # {resource_name} -> 靜默
SNOOZE_TTL_SECONDS = 30 * 60 # 30 分鐘
SILENCE_TTL_SECONDS = 60 * 60 # 1 小時
# 2026-04-01 Claude Code: Long Polling 分散式 Leader Election
# 防止多 Pod 同時 getUpdates → 409 Conflict 互搶問題
POLLING_LEADER_KEY = "telegram:polling:leader"
POLLING_LEADER_TTL = 45 # seconds - Pod 宕掉後 45s 自動轉移
POLLING_LEADER_RENEW = 20 # seconds - 每 20s 續約
POLLING_LEADER_WATCH = 30 # seconds - 非 Leader Pod 每 30s 嘗試接管
logger = structlog.get_logger(__name__)
# =============================================================================
@@ -1018,6 +1026,9 @@ class TelegramGateway:
self._polling_active = False
self._polling_task: asyncio.Task | None = None
self._last_update_id = 0
# 2026-04-01 Claude Code: 分散式 Leader Election (防 2-Pod 409 互搶)
self._pod_id = os.environ.get("POD_NAME", os.urandom(8).hex())
self._leader_task: asyncio.Task | None = None
# Phase 6.5: 心跳監控 (防止沉默盲點)
self._last_message_time: datetime | None = None
self._heartbeat_task: asyncio.Task | None = None
@@ -2232,15 +2243,17 @@ class TelegramGateway:
async def close(self) -> None:
"""關閉 Gateway"""
# 停止 Long Polling
# 停止 Long Polling 與 Leader 相關 Tasks
self._polling_active = False
if self._polling_task and not self._polling_task.done():
self._polling_task.cancel()
try:
await self._polling_task
except asyncio.CancelledError:
pass
self._polling_task = None
for task in (self._polling_task, self._leader_task):
if task and not task.done():
task.cancel()
try:
await task
except asyncio.CancelledError:
pass
self._polling_task = None
self._leader_task = None
if self._http_client:
await self._http_client.aclose()
@@ -2258,6 +2271,9 @@ class TelegramGateway:
取代 Webhook 模式,適用於內網環境
統帥鐵律: 內網無法接收外部 Webhook必須主動輪詢
2026-04-01 Claude Code: 加入 Redis Leader Election
多 Pod 環境下,只有 Leader 執行 getUpdates其餘 Pod 進入 Watcher 模式
"""
if not self._initialized:
success = await self.initialize()
@@ -2269,15 +2285,33 @@ class TelegramGateway:
logger.warning("telegram_long_polling_already_running")
return
# 先刪除 Webhook確保可以使用 getUpdates
# 嘗試取得 Leader Lock (NX = 僅在不存在時設定)
redis = await get_redis()
acquired = await redis.set(POLLING_LEADER_KEY, self._pod_id, nx=True, ex=POLLING_LEADER_TTL)
if not acquired:
current_leader = await redis.get(POLLING_LEADER_KEY)
logger.info(
"telegram_polling_not_leader",
pod_id=self._pod_id,
current_leader=current_leader,
action="watcher_mode",
)
# 啟動 Watcher定期嘗試接管
self._leader_task = asyncio.create_task(self._leader_watcher())
return
# 取得 Leader Lock開始 Polling
await self._delete_webhook()
self._polling_active = True
self._last_update_id = 0
self._polling_task = asyncio.create_task(self._polling_loop())
self._leader_task = asyncio.create_task(self._leader_renewer())
logger.info(
"telegram_long_polling_started",
pod_id=self._pod_id,
timeout=LONG_POLLING_TIMEOUT,
chat_id=self.chat_id[:10] + "..." if self.chat_id else "N/A",
)
@@ -2354,15 +2388,23 @@ class TelegramGateway:
except httpx.HTTPStatusError as e:
if e.response.status_code == 409:
# 409 Conflict: 另一個 getUpdates session 殘留
# 可能是舊 pod/host 的連線未清除,等待後重試
# 409 Conflict: 另一個 Pod 正在 polling主動釋放 Leader Lock
# 2026-04-01 Claude Code: 改為釋放 Lock 讓 Watcher 競爭
# (舊: 侵略性搶佔 2s已不適用 - 現在是多 Pod 場景而非 .188 搶佔)
logger.warning(
"telegram_polling_conflict",
status=409,
message="409 衝突,等待殘留 session 過期...",
pod_id=self._pod_id,
action="releasing_leader_lock",
)
# 2026-03-31 ogt: 侵略性搶佔模式 - 縮短等待以從 .188 手中奪回主權
await asyncio.sleep(2)
redis = await get_redis()
current = await redis.get(POLLING_LEADER_KEY)
if current == self._pod_id:
await redis.delete(POLLING_LEADER_KEY)
self._polling_active = False
# Watcher 會在 POLLING_LEADER_WATCH 秒後重新競爭
self._leader_task = asyncio.create_task(self._leader_watcher())
break
else:
logger.error("telegram_polling_http_error", status=e.response.status_code)
await asyncio.sleep(LONG_POLLING_RETRY_DELAY)
@@ -2374,6 +2416,68 @@ class TelegramGateway:
logger.info("telegram_long_polling_stopped")
async def _leader_renewer(self) -> None:
"""
Leader Lock 續約背景任務
每 POLLING_LEADER_RENEW 秒更新 Redis TTL
確保 Leader 在 Poll 期間持續持有 Lock。
若 Lock 被搶走,停止 Polling。
2026-04-01 Claude Code: 分散式 Leader Election
"""
while self._polling_active:
await asyncio.sleep(POLLING_LEADER_RENEW)
if not self._polling_active:
break
try:
redis = await get_redis()
current = await redis.get(POLLING_LEADER_KEY)
if current != self._pod_id:
logger.warning(
"telegram_leader_lock_lost",
pod_id=self._pod_id,
current_leader=current,
)
self._polling_active = False
break
await redis.expire(POLLING_LEADER_KEY, POLLING_LEADER_TTL)
except Exception as e:
logger.error("telegram_leader_renew_error", error=str(e))
async def _leader_watcher(self) -> None:
"""
非 Leader Pod 的接管監控任務
每 POLLING_LEADER_WATCH 秒嘗試取得 Leader Lock。
若原 Leader 宕掉TTL 過期),此 Pod 接管 Polling。
2026-04-01 Claude Code: 分散式 Leader Election
"""
while not self._polling_active:
await asyncio.sleep(POLLING_LEADER_WATCH)
try:
redis = await get_redis()
acquired = await redis.set(
POLLING_LEADER_KEY, self._pod_id, nx=True, ex=POLLING_LEADER_TTL
)
if acquired:
logger.info(
"telegram_leader_acquired",
pod_id=self._pod_id,
action="starting_polling",
)
await self._delete_webhook()
self._polling_active = True
self._last_update_id = 0
self._polling_task = asyncio.create_task(self._polling_loop())
self._leader_task = asyncio.create_task(self._leader_renewer())
break
except asyncio.CancelledError:
break
except Exception as e:
logger.error("telegram_leader_watch_error", error=str(e))
async def _get_updates(self) -> list[dict]:
"""
呼叫 Telegram getUpdates API