feat(api): ADR-038 Circuit Breaker 整合 + Graceful Degradation

sentry_webhook.py:
- 整合 OpenClawGuard (Circuit Breaker + Semaphore)
- 斷路狀態快速失敗,不呼叫 OpenClaw
- 並發控制: 最多 3 個同時 LLM 推理

anomaly_counter.py:
- record_anomaly() Redis 故障 Graceful Degradation
- 失敗時返回預設 AnomalyFrequency (count=0)
- 不中斷主流程

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
OG T
2026-03-29 15:55:51 +08:00
parent 39396dc57a
commit 89a2339796
2 changed files with 83 additions and 23 deletions

View File

@@ -21,6 +21,7 @@ import structlog
from fastapi import APIRouter, BackgroundTasks, HTTPException, Request
from pydantic import BaseModel
from src.core.circuit_breaker import get_openclaw_guard
from src.core.config import settings
from src.core.metrics import (
record_alert_chain_failure,
@@ -280,33 +281,52 @@ async def call_openclaw_analyzer(error_context: dict) -> ErrorAnalysisResult | N
"""
呼叫 OpenClaw Error Analyzer Agent
ADR-038: 雙層保護
- Layer 1: Circuit Breaker5 連續失敗 → 斷路 60 秒)
- Layer 2: Concurrency Semaphore最多 3 並發)
優先使用 Ollama (本地,零成本)
Fallback: Claude (高嚴重性)
"""
try:
async with httpx.AsyncClient(timeout=60.0) as client:
response = await client.post(
f"{settings.OPENCLAW_URL}/api/v1/analyze/error",
json={
"error_context": error_context,
"prefer_local": True, # 優先 Ollama
}
)
guard = get_openclaw_guard()
if response.status_code == 200:
data = response.json()
return ErrorAnalysisResult(**data)
else:
logger.warning(f"OpenClaw returned {response.status_code}")
return None
# ADR-038 Layer 1: Circuit Breaker 快速失敗
if guard.is_circuit_open():
logger.warning(
"openclaw_circuit_open",
metrics=guard.get_metrics(),
)
return None
except httpx.TimeoutException:
logger.warning("OpenClaw analysis timeout, trying fallback prompt")
# TODO: 直接呼叫 Ollama/Claude 作為 fallback
return None
except Exception as e:
logger.exception(f"OpenClaw call failed: {e}")
return None
# ADR-038 Layer 2: Concurrency Semaphore 排隊
async with guard.semaphore:
try:
async with httpx.AsyncClient(timeout=60.0) as client:
response = await client.post(
f"{settings.OPENCLAW_URL}/api/v1/analyze/error",
json={
"error_context": error_context,
"prefer_local": True, # 優先 Ollama
}
)
if response.status_code == 200:
data = response.json()
guard.record_success()
return ErrorAnalysisResult(**data)
else:
logger.warning(f"OpenClaw returned {response.status_code}")
guard.record_failure()
return None
except httpx.TimeoutException:
logger.warning("OpenClaw analysis timeout")
guard.record_failure()
return None
except Exception as e:
logger.exception(f"OpenClaw call failed: {e}")
guard.record_failure()
return None
async def post_sentry_comment(