feat(api): ADR-038 Circuit Breaker 整合 + Graceful Degradation
sentry_webhook.py: - 整合 OpenClawGuard (Circuit Breaker + Semaphore) - 斷路狀態快速失敗,不呼叫 OpenClaw - 並發控制: 最多 3 個同時 LLM 推理 anomaly_counter.py: - record_anomaly() Redis 故障 Graceful Degradation - 失敗時返回預設 AnomalyFrequency (count=0) - 不中斷主流程 Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
@@ -21,6 +21,7 @@ import structlog
|
||||
from fastapi import APIRouter, BackgroundTasks, HTTPException, Request
|
||||
from pydantic import BaseModel
|
||||
|
||||
from src.core.circuit_breaker import get_openclaw_guard
|
||||
from src.core.config import settings
|
||||
from src.core.metrics import (
|
||||
record_alert_chain_failure,
|
||||
@@ -280,33 +281,52 @@ async def call_openclaw_analyzer(error_context: dict) -> ErrorAnalysisResult | N
|
||||
"""
|
||||
呼叫 OpenClaw Error Analyzer Agent
|
||||
|
||||
ADR-038: 雙層保護
|
||||
- Layer 1: Circuit Breaker(5 連續失敗 → 斷路 60 秒)
|
||||
- Layer 2: Concurrency Semaphore(最多 3 並發)
|
||||
|
||||
優先使用 Ollama (本地,零成本)
|
||||
Fallback: Claude (高嚴重性)
|
||||
"""
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=60.0) as client:
|
||||
response = await client.post(
|
||||
f"{settings.OPENCLAW_URL}/api/v1/analyze/error",
|
||||
json={
|
||||
"error_context": error_context,
|
||||
"prefer_local": True, # 優先 Ollama
|
||||
}
|
||||
)
|
||||
guard = get_openclaw_guard()
|
||||
|
||||
if response.status_code == 200:
|
||||
data = response.json()
|
||||
return ErrorAnalysisResult(**data)
|
||||
else:
|
||||
logger.warning(f"OpenClaw returned {response.status_code}")
|
||||
return None
|
||||
# ADR-038 Layer 1: Circuit Breaker 快速失敗
|
||||
if guard.is_circuit_open():
|
||||
logger.warning(
|
||||
"openclaw_circuit_open",
|
||||
metrics=guard.get_metrics(),
|
||||
)
|
||||
return None
|
||||
|
||||
except httpx.TimeoutException:
|
||||
logger.warning("OpenClaw analysis timeout, trying fallback prompt")
|
||||
# TODO: 直接呼叫 Ollama/Claude 作為 fallback
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.exception(f"OpenClaw call failed: {e}")
|
||||
return None
|
||||
# ADR-038 Layer 2: Concurrency Semaphore 排隊
|
||||
async with guard.semaphore:
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=60.0) as client:
|
||||
response = await client.post(
|
||||
f"{settings.OPENCLAW_URL}/api/v1/analyze/error",
|
||||
json={
|
||||
"error_context": error_context,
|
||||
"prefer_local": True, # 優先 Ollama
|
||||
}
|
||||
)
|
||||
|
||||
if response.status_code == 200:
|
||||
data = response.json()
|
||||
guard.record_success()
|
||||
return ErrorAnalysisResult(**data)
|
||||
else:
|
||||
logger.warning(f"OpenClaw returned {response.status_code}")
|
||||
guard.record_failure()
|
||||
return None
|
||||
|
||||
except httpx.TimeoutException:
|
||||
logger.warning("OpenClaw analysis timeout")
|
||||
guard.record_failure()
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.exception(f"OpenClaw call failed: {e}")
|
||||
guard.record_failure()
|
||||
return None
|
||||
|
||||
|
||||
async def post_sentry_comment(
|
||||
|
||||
@@ -132,14 +132,54 @@ class AnomalyCounter:
|
||||
"""
|
||||
記錄一次異常發生
|
||||
|
||||
ADR-038/039: Redis 故障時 Graceful Degradation
|
||||
- 記錄錯誤但不中斷主流程
|
||||
- 返回空的 AnomalyFrequency(頻率計數 = 0)
|
||||
|
||||
Args:
|
||||
anomaly_signature: 異常簽名字典
|
||||
|
||||
Returns:
|
||||
AnomalyFrequency: 當前頻率統計
|
||||
AnomalyFrequency: 當前頻率統計(Redis 失敗時返回預設值)
|
||||
"""
|
||||
anomaly_key = self.hash_signature(anomaly_signature)
|
||||
now = datetime.now()
|
||||
|
||||
try:
|
||||
return await self._record_anomaly_impl(
|
||||
anomaly_key=anomaly_key,
|
||||
anomaly_signature=anomaly_signature,
|
||||
now=now,
|
||||
)
|
||||
except Exception as e:
|
||||
# ADR-038: Redis 故障 Graceful Degradation
|
||||
logger.warning(
|
||||
"anomaly_counter_redis_error",
|
||||
error=str(e),
|
||||
anomaly_key=anomaly_key,
|
||||
fallback="returning_default_frequency",
|
||||
)
|
||||
# 返回預設值,不阻擋主流程
|
||||
return AnomalyFrequency(
|
||||
anomaly_key=anomaly_key,
|
||||
count_1h=0,
|
||||
count_24h=0,
|
||||
count_7d=0,
|
||||
count_30d=0,
|
||||
first_seen=now,
|
||||
last_seen=now,
|
||||
auto_repair_count=0,
|
||||
permanent_fix_applied=False,
|
||||
escalation_level=None,
|
||||
)
|
||||
|
||||
async def _record_anomaly_impl(
|
||||
self,
|
||||
anomaly_key: str,
|
||||
anomaly_signature: dict,
|
||||
now: datetime,
|
||||
) -> AnomalyFrequency:
|
||||
"""實際的異常記錄邏輯(可能拋出 Redis 異常)"""
|
||||
timestamp = now.timestamp()
|
||||
timeline_key = f"{self.PREFIX_TIMELINE}{anomaly_key}"
|
||||
|
||||
|
||||
Reference in New Issue
Block a user