fix(aiops): suppress repeated llm alert loops

apps/api/src/api/v1/webhooks.py

@@ -53,6 +53,10 @@ from src.models.approval import (
 # [首席架構師] 移除 generate_alert_fingerprint 直接 import，改用 AlertAnalyzer.generate_fingerprint v1.2 2026-04-01 Asia/Taipei
 from src.models.webhook import AlertPayload, AlertResponse
 from src.services.alert_analyzer_service import AlertAnalyzer
+from src.services.alertmanager_llm_guard import (
+    ALERTMANAGER_LLM_INFLIGHT_LOCK_TTL_SECONDS,
+    try_acquire_alertmanager_llm_lock,
+)
 from src.services.approval_db import get_approval_service
 
 # Phase 17 P0: Service 層 (消除 Router 直接存取 Redis)
@@ -2150,12 +2154,22 @@ async def alertmanager_webhook(
     # 2026-04-14 Claude Haiku 4.5 Asia/Taipei
     # 位置：指紋生成後、LLM 分析前（短路子告警）
     # ==========================================================================
-    grouping_result = await get_alert_grouping_service().evaluate(
-        alertname=alertname,
-        namespace=namespace,
-        fingerprint=fingerprint,
-    )
-    if grouping_result.is_grouped:
+    try:
+        grouping_result = await get_alert_grouping_service().evaluate(
+            alertname=alertname,
+            namespace=namespace,
+            fingerprint=fingerprint,
+        )
+    except Exception as e:
+        grouping_result = None
+        logger.warning(
+            "alertmanager_grouping_failed_fail_open",
+            alert_id=alert_id,
+            fingerprint=fingerprint,
+            error=str(e),
+        )
+
+    if grouping_result and grouping_result.is_grouped:
         logger.info(
             "alertmanager_grouped_skip",
             alert_id=alert_id,
@@ -2258,6 +2272,21 @@ async def alertmanager_webhook(
                 approval_created=False,
             )
 
+        if not await try_acquire_alertmanager_llm_lock(fingerprint, alert_id):
+            logger.info(
+                "alertmanager_llm_inflight_suppressed",
+                alert_id=alert_id,
+                fingerprint=fingerprint,
+                ttl_seconds=ALERTMANAGER_LLM_INFLIGHT_LOCK_TTL_SECONDS,
+            )
+            return AlertResponse(
+                success=True,
+                message="🛡️ 告警已由同指紋背景 AI 分析處理中，跳過重複 LLM 呼叫",
+                alert_id=alert_id,
+                approval_created=False,
+                converged=True,
+            )
+
         # ==========================================================================
         # ADR-089 (2026-04-17 ogt + Claude Sonnet 4.6): 新告警 — 背景 LLM 分析
         # 立即回傳 202，AI 辯證在背景非同步執行
@@ -2271,6 +2300,7 @@ async def alertmanager_webhook(
             "source": "alertmanager",
             "target_resource": target_resource,
             "namespace": namespace,
+            "fingerprint": fingerprint,
             "message": message,
             "annotations": dict(alert.annotations) if alert.annotations else {},
             "metrics": {},
@@ -2303,11 +2333,18 @@ async def alertmanager_webhook(
         )
 
     except Exception as e:
-        logger.error("alertmanager_error", error=str(e))
-        raise HTTPException(
-            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
-            detail=f"Failed to process alert: {str(e)}",
-        ) from e
+        logger.error(
+            "alertmanager_degraded_accepted_no_retry",
+            alert_id=alert_id,
+            fingerprint=fingerprint,
+            error=str(e),
+        )
+        return AlertResponse(
+            success=False,
+            message="⚠️ 告警已接收但處理降級，避免 Alertmanager retry storm；已交由背景治理/人工介入追蹤",
+            alert_id=alert_id,
+            approval_created=False,
+        )
 
 
 @router.get(

apps/api/src/services/alertmanager_llm_guard.py

@@ -0,0 +1,45 @@
+"""Alertmanager LLM storm guards.
+
+Service-layer Redis helpers used by webhook routers to avoid spawning duplicate
+LLM analysis tasks for the same Alertmanager fingerprint.
+"""
+
+from src.core.logging import get_logger
+from src.core.redis_client import get_redis
+
+logger = get_logger("awoooi.alertmanager_llm_guard")
+
+ALERTMANAGER_LLM_INFLIGHT_LOCK_TTL_SECONDS = 600
+
+
+def alertmanager_llm_inflight_key(fingerprint: str) -> str:
+    """Return the Redis lock key for one Alertmanager fingerprint entering AI analysis."""
+
+    return f"alertmanager:llm_inflight:{fingerprint}"
+
+
+async def try_acquire_alertmanager_llm_lock(
+    fingerprint: str,
+    alert_id: str,
+    *,
+    ttl_seconds: int = ALERTMANAGER_LLM_INFLIGHT_LOCK_TTL_SECONDS,
+) -> bool:
+    """Prevent same-second duplicate Alertmanager deliveries from spawning LLM calls."""
+
+    try:
+        redis = get_redis()
+        acquired = await redis.set(
+            alertmanager_llm_inflight_key(fingerprint),
+            alert_id,
+            ex=ttl_seconds,
+            nx=True,
+        )
+        return bool(acquired)
+    except Exception as exc:
+        logger.warning(
+            "alertmanager_llm_inflight_lock_failed_fail_open",
+            fingerprint=fingerprint,
+            alert_id=alert_id,
+            error=str(exc),
+        )
+        return True

apps/api/src/services/openclaw.py

@@ -124,6 +124,21 @@ def _backfill_kubectl_command(proposal: dict, tools: list) -> None:
 # OpenClaw Service
 # =============================================================================
 
+def _build_alert_cache_context_hash(alert_context: dict | None) -> str:
+    """Build a stable LLM cache scope for repeat alerts without dynamic annotations."""
+
+    if not alert_context:
+        return ""
+
+    alertname = alert_context.get("alertname") or alert_context.get("alert_type", "")
+    category = alert_context.get("alert_category", "")
+    namespace = alert_context.get("namespace", "")
+    target = alert_context.get("target_resource", "")
+    severity = alert_context.get("severity", "")
+    fingerprint = alert_context.get("fingerprint", "")
+    return f"{alertname}:{category}:{namespace}:{target}:{severity}:{fingerprint}"
+
+
 class OpenClawService:
     """
     OpenClaw AI 決策服務 - True LLM + SignOz Integration
@@ -727,9 +742,19 @@ class OpenClawService:
         """
         生成 LLM 快取鍵
 
-        使用 prompt 內容的 SHA256 作為快取鍵，確保相同問題不重複呼叫 LLM
+        有告警上下文時，使用 prompt family + 穩定告警維度，避免 annotations /
+        SignOz 即時數值讓同一告警每 20 秒打穿快取；沒有上下文時仍用完整 prompt。
         """
-        content = f"{prompt}:{context_hash}"
+        if context_hash:
+            prompt_family_source = (
+                "openclaw_alert_analysis"
+                if "## Alert Data:" in prompt
+                else prompt[:512]
+            )
+            prompt_family = hashlib.sha256(prompt_family_source.encode()).hexdigest()[:8]
+            content = f"{prompt_family}:{context_hash}"
+        else:
+            content = prompt
         hash_digest = hashlib.sha256(content.encode()).hexdigest()[:16]
         return f"llm_cache:{hash_digest}"
 
@@ -760,12 +785,7 @@ class OpenClawService:
         # 2026-04-16 ogt + Claude Sonnet 4.6: 修復 — alertname 才是主要識別符
         # 舊版用 alert_type:target_resource → 不同告警 (e.g. PostgreSQLDiskGrowth vs PodCrashLoop)
         # 在 alert_type="custom" 時共用同一快取鍵 → 全部回傳相同 LLM 結果
-        context_hash = ""
-        if alert_context:
-            # alertname 優先；無 alertname 時 fallback 到 alert_type
-            _alertname = alert_context.get("alertname") or alert_context.get("alert_type", "")
-            _target = alert_context.get("target_resource", "")
-            context_hash = f"{_alertname}:{_target}"
+        context_hash = _build_alert_cache_context_hash(alert_context)
 
         cache_key = self._generate_cache_key(prompt, context_hash)
 

apps/api/tests/test_alertmanager_rule_bypass.py

@@ -4,6 +4,10 @@ from src.api.v1.webhooks import (
     _should_bypass_alertmanager_llm,
     _should_use_alertmanager_rule_first,
 )
+from src.services.alertmanager_llm_guard import (
+    ALERTMANAGER_LLM_INFLIGHT_LOCK_TTL_SECONDS,
+    alertmanager_llm_inflight_key,
+)
 from src.services.decision_manager import (
     _is_host_layer_ssh_category,
     _is_non_k8s_host_category,
@@ -100,6 +104,13 @@ def test_backup_failure_blocks_k8s_auto_execute():
     assert _is_non_k8s_host_category("infrastructure") is False
 
 
+def test_alertmanager_llm_inflight_lock_key_is_fingerprint_scoped():
+    fingerprint = "abc123"
+
+    assert alertmanager_llm_inflight_key(fingerprint) == "alertmanager:llm_inflight:abc123"
+    assert ALERTMANAGER_LLM_INFLIGHT_LOCK_TTL_SECONDS == 600
+
+
 def test_resolved_guard_stamp_without_timestamp_is_clean():
     assert _format_resolved_guard_stamp(None) == "✅ 此事件已解決"
 

apps/api/tests/test_openclaw_cache_key.py

@@ -0,0 +1,42 @@
+from src.services.openclaw import OpenClawService, _build_alert_cache_context_hash
+
+
+def test_openclaw_cache_key_uses_stable_alert_scope_when_context_hash_exists():
+    service = object.__new__(OpenClawService)
+    context_hash = "HostBackupFailed:backup_failure:awoooi-prod:awoooi-frequent:critical:fp-1"
+    prompt_a = "System prompt and instructions\n\n## Alert Data:\ncurrent_value=1"
+    prompt_b = "System prompt and instructions\n\n## Alert Data:\ncurrent_value=2"
+
+    assert service._generate_cache_key(prompt_a, context_hash) == service._generate_cache_key(
+        prompt_b,
+        context_hash,
+    )
+
+
+def test_openclaw_cache_key_keeps_full_prompt_specificity_without_context_hash():
+    service = object.__new__(OpenClawService)
+    prompt_a = "System prompt and instructions\n\n## Alert Data:\ncurrent_value=1"
+    prompt_b = "System prompt and instructions\n\n## Alert Data:\ncurrent_value=2"
+
+    assert service._generate_cache_key(prompt_a) != service._generate_cache_key(prompt_b)
+
+
+def test_openclaw_alert_cache_context_hash_ignores_dynamic_annotations():
+    base_context = {
+        "alertname": "HostBackupFailed",
+        "alert_category": "backup_failure",
+        "namespace": "awoooi-prod",
+        "target_resource": "awoooi-frequent",
+        "severity": "critical",
+        "fingerprint": "fp-1",
+        "annotations": {"description": "failed at 14:00"},
+    }
+    next_context = {
+        **base_context,
+        "annotations": {"description": "failed at 14:01"},
+        "message": "new volatile message",
+    }
+
+    assert _build_alert_cache_context_hash(base_context) == _build_alert_cache_context_hash(
+        next_context
+    )