""" 全域修復熔斷機制 ================ ADR-039:防止跨資源循環修復 設計原則: - Redis TTL 滑動窗口(15 分鐘) - 失敗降級:Redis 故障時保守跳過自動修復,強制人工確認 遵循 leWOOOgo 積木化鐵律: - 此模組屬於 Service 層 - 依賴 core/redis_client - 無副作用的純邏輯判斷 2026-03-29 ogt - ADR-039 實作 """ import structlog from src.core.redis_client import get_redis logger = structlog.get_logger(__name__) GLOBAL_COOLDOWN_KEY = "global:auto_repair:count" GLOBAL_COOLDOWN_TTL = 900 # 15 分鐘窗口 GLOBAL_COOLDOWN_THRESHOLD = 5 # 超過 5 次強制凍結 # ADR-039: 有狀態服務黑名單(永遠禁止自動重啟) STATEFUL_SERVICE_BLACKLIST = frozenset( { # PostgreSQL "postgres", "postgresql", "awoooi-postgres", # Redis "redis", "awoooi-redis", "redis-stack", # ClickHouse (SignOz) "clickhouse", "signoz-clickhouse", # 其他有狀態服務 "elasticsearch", "etcd", "minio", "awoooi-minio", "kafka", "zookeeper", } ) async def check_global_repair_cooldown( incident_id: str, affected_services: list[str] | None = None, ) -> tuple[bool, str]: """ 檢查是否允許自動修復 Args: incident_id: 事件 ID(用於日誌追蹤) affected_services: 受影響的服務列表 Returns: (can_repair: bool, reason: str) """ affected_services = affected_services or [] # === 硬禁令:有狀態服務黑名單 (純邏輯,無需 Redis) === for service in affected_services: service_lower = service.lower() for blacklisted in STATEFUL_SERVICE_BLACKLIST: if blacklisted in service_lower: reason = f"服務 {service} 為有狀態服務,禁止自動重啟,請統帥手動介入" logger.warning( "stateful_service_blocked", service=service, incident_id=incident_id, blacklist_match=blacklisted, ) return False, reason # === 全域冷卻期:Redis 計數 === # 2026-04-01 ogt: 將 get_redis() 移入 try-except,防止 Redis 未初始化時拋出未捕獲例外 try: redis = get_redis() count_raw = await redis.get(GLOBAL_COOLDOWN_KEY) current_count = int(count_raw) if count_raw else 0 if current_count >= GLOBAL_COOLDOWN_THRESHOLD: reason = ( f"系統在過去 15 分鐘內已自動修復 {current_count} 次," f"超出安全閾值 {GLOBAL_COOLDOWN_THRESHOLD}," "強制轉為人工審核模式" ) logger.warning( "global_repair_cooldown_active", current_count=current_count, threshold=GLOBAL_COOLDOWN_THRESHOLD, incident_id=incident_id, ) return False, reason return True, "允許自動修復" except Exception as e: # Redis 故障 → 保守策略:禁止自動修復 logger.error( "global_repair_cooldown_redis_error", error=str(e), fallback="blocking_auto_repair_for_safety", incident_id=incident_id, ) return False, f"Redis 連線異常,保守禁止自動修復(原因:{e})" async def record_global_repair_action() -> None: """ 記錄一次全域修復動作 使用 INCR + EXPIRE 實現滑動窗口計數 注意:INCR 是原子操作,多個 Worker 並發安全 """ try: redis = get_redis() count = await redis.incr(GLOBAL_COOLDOWN_KEY) # 只在第一次設定 TTL(避免頻繁重設導致窗口延長) if count == 1: await redis.expire(GLOBAL_COOLDOWN_KEY, GLOBAL_COOLDOWN_TTL) logger.info( "global_repair_action_recorded", count=count, threshold=GLOBAL_COOLDOWN_THRESHOLD, ttl_seconds=GLOBAL_COOLDOWN_TTL, ) except Exception as e: # Redis 故障:靜默失敗(不影響主流程) logger.warning("global_repair_record_failed", error=str(e)) async def get_global_repair_status() -> dict: """ 取得全域修復狀態(供 API/監控用) Returns: { "current_count": int, "threshold": int, "is_frozen": bool, "ttl_remaining": int | None, } """ try: redis = get_redis() count_raw = await redis.get(GLOBAL_COOLDOWN_KEY) current_count = int(count_raw) if count_raw else 0 ttl = await redis.ttl(GLOBAL_COOLDOWN_KEY) return { "current_count": current_count, "threshold": GLOBAL_COOLDOWN_THRESHOLD, "is_frozen": current_count >= GLOBAL_COOLDOWN_THRESHOLD, "ttl_remaining": ttl if ttl > 0 else None, "window_seconds": GLOBAL_COOLDOWN_TTL, } except Exception as e: logger.warning("get_global_repair_status_failed", error=str(e)) return { "current_count": -1, "threshold": GLOBAL_COOLDOWN_THRESHOLD, "is_frozen": True, # 保守假設:凍結 "ttl_remaining": None, "error": str(e), }