diff --git a/apps/api/src/api/v1/sentry_webhook.py b/apps/api/src/api/v1/sentry_webhook.py index a58083ccd..4ace372de 100644 --- a/apps/api/src/api/v1/sentry_webhook.py +++ b/apps/api/src/api/v1/sentry_webhook.py @@ -21,6 +21,7 @@ import structlog from fastapi import APIRouter, BackgroundTasks, HTTPException, Request from pydantic import BaseModel +from src.core.circuit_breaker import get_openclaw_guard from src.core.config import settings from src.core.metrics import ( record_alert_chain_failure, @@ -280,33 +281,52 @@ async def call_openclaw_analyzer(error_context: dict) -> ErrorAnalysisResult | N """ 呼叫 OpenClaw Error Analyzer Agent + ADR-038: 雙層保護 + - Layer 1: Circuit Breaker(5 連續失敗 → 斷路 60 秒) + - Layer 2: Concurrency Semaphore(最多 3 並發) + 優先使用 Ollama (本地,零成本) Fallback: Claude (高嚴重性) """ - try: - async with httpx.AsyncClient(timeout=60.0) as client: - response = await client.post( - f"{settings.OPENCLAW_URL}/api/v1/analyze/error", - json={ - "error_context": error_context, - "prefer_local": True, # 優先 Ollama - } - ) + guard = get_openclaw_guard() - if response.status_code == 200: - data = response.json() - return ErrorAnalysisResult(**data) - else: - logger.warning(f"OpenClaw returned {response.status_code}") - return None + # ADR-038 Layer 1: Circuit Breaker 快速失敗 + if guard.is_circuit_open(): + logger.warning( + "openclaw_circuit_open", + metrics=guard.get_metrics(), + ) + return None - except httpx.TimeoutException: - logger.warning("OpenClaw analysis timeout, trying fallback prompt") - # TODO: 直接呼叫 Ollama/Claude 作為 fallback - return None - except Exception as e: - logger.exception(f"OpenClaw call failed: {e}") - return None + # ADR-038 Layer 2: Concurrency Semaphore 排隊 + async with guard.semaphore: + try: + async with httpx.AsyncClient(timeout=60.0) as client: + response = await client.post( + f"{settings.OPENCLAW_URL}/api/v1/analyze/error", + json={ + "error_context": error_context, + "prefer_local": True, # 優先 Ollama + } + ) + + if response.status_code == 200: + data = response.json() + guard.record_success() + return ErrorAnalysisResult(**data) + else: + logger.warning(f"OpenClaw returned {response.status_code}") + guard.record_failure() + return None + + except httpx.TimeoutException: + logger.warning("OpenClaw analysis timeout") + guard.record_failure() + return None + except Exception as e: + logger.exception(f"OpenClaw call failed: {e}") + guard.record_failure() + return None async def post_sentry_comment( diff --git a/apps/api/src/services/anomaly_counter.py b/apps/api/src/services/anomaly_counter.py index 94f3a433e..17a687a13 100644 --- a/apps/api/src/services/anomaly_counter.py +++ b/apps/api/src/services/anomaly_counter.py @@ -132,14 +132,54 @@ class AnomalyCounter: """ 記錄一次異常發生 + ADR-038/039: Redis 故障時 Graceful Degradation + - 記錄錯誤但不中斷主流程 + - 返回空的 AnomalyFrequency(頻率計數 = 0) + Args: anomaly_signature: 異常簽名字典 Returns: - AnomalyFrequency: 當前頻率統計 + AnomalyFrequency: 當前頻率統計(Redis 失敗時返回預設值) """ anomaly_key = self.hash_signature(anomaly_signature) now = datetime.now() + + try: + return await self._record_anomaly_impl( + anomaly_key=anomaly_key, + anomaly_signature=anomaly_signature, + now=now, + ) + except Exception as e: + # ADR-038: Redis 故障 Graceful Degradation + logger.warning( + "anomaly_counter_redis_error", + error=str(e), + anomaly_key=anomaly_key, + fallback="returning_default_frequency", + ) + # 返回預設值,不阻擋主流程 + return AnomalyFrequency( + anomaly_key=anomaly_key, + count_1h=0, + count_24h=0, + count_7d=0, + count_30d=0, + first_seen=now, + last_seen=now, + auto_repair_count=0, + permanent_fix_applied=False, + escalation_level=None, + ) + + async def _record_anomaly_impl( + self, + anomaly_key: str, + anomaly_signature: dict, + now: datetime, + ) -> AnomalyFrequency: + """實際的異常記錄邏輯(可能拋出 Redis 異常)""" timestamp = now.timestamp() timeline_key = f"{self.PREFIX_TIMELINE}{anomaly_key}"