# AWOOOI 告警鏈路監控
# 負責人: DevOps Commander
# 版本: v1.0
# 日期: 2026-03-29
# ADR: ADR-037 (監控增強架構) Wave B.1
#
# 用途: 監控告警鏈路健康狀態，防止 2026-03-26 路徑錯誤導致無告警事故重演

apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: alert-chain-monitor
  namespace: monitoring
  labels:
    release: prometheus
    app: prometheus
spec:
  groups:
    # =========================================================================
    # 告警鏈路健康監控
    # =========================================================================
    - name: alert_chain
      rules:
        # -----------------------------------------------------------------
        # Alertmanager Webhook 鏈路異常
        # -----------------------------------------------------------------
        - alert: AlertChainBroken_Alertmanager
          expr: |
            sum(rate(awoooi_webhook_requests_total{
              source="alertmanager",
              status!="success"
            }[5m])) / sum(rate(awoooi_webhook_requests_total{
              source="alertmanager"
            }[5m])) > 0.1
          for: 10m
          labels:
            severity: critical
            service: alert-chain
            team: platform
          annotations:
            summary: "Alertmanager Webhook 錯誤率 > 10%"
            description: "告警可能無法正確送達 AWOOOI API"
            runbook_url: "https://awoooi.internal/runbooks/alert-chain"

        # -----------------------------------------------------------------
        # Sentry Webhook 鏈路異常
        # -----------------------------------------------------------------
        - alert: AlertChainBroken_Sentry
          expr: |
            sum(rate(awoooi_webhook_requests_total{
              source="sentry",
              status!="success"
            }[5m])) / sum(rate(awoooi_webhook_requests_total{
              source="sentry"
            }[5m])) > 0.1
          for: 10m
          labels:
            severity: warning
            service: alert-chain
            team: platform
          annotations:
            summary: "Sentry Webhook 錯誤率 > 10%"
            description: "Sentry 錯誤可能無法正確處理"

        # -----------------------------------------------------------------
        # SignOz Webhook 鏈路異常
        # -----------------------------------------------------------------
        - alert: AlertChainBroken_SignOz
          expr: |
            sum(rate(awoooi_webhook_requests_total{
              source="signoz",
              status!="success"
            }[5m])) / sum(rate(awoooi_webhook_requests_total{
              source="signoz"
            }[5m])) > 0.1
          for: 10m
          labels:
            severity: warning
            service: alert-chain
            team: platform
          annotations:
            summary: "SignOz Webhook 錯誤率 > 10%"
            description: "SignOz 告警可能無法正確處理"

        # -----------------------------------------------------------------
        # 告警鏈路完全斷裂 (無任何告警)
        # P1-1 修復: 按 source 分組 (2026-03-29)
        # -----------------------------------------------------------------
        - alert: NoAlertsReceived2Hours
          expr: |
            time() - max by (source)(awoooi_alert_chain_last_success_timestamp{source="alertmanager"}) > 7200
          for: 5m
          labels:
            severity: warning
            service: alert-chain
            team: platform
          annotations:
            summary: "Alertmanager 主鏈路 2 小時內未收到告警"
            description: "Alertmanager 是固定主鏈路；Sentry/SignOz 沉默不代表鏈路故障，錯誤率另有 AlertChainBroken_* 規則監控。請執行 Smoke Test: python scripts/alert_chain_smoke_test.py"

        # -----------------------------------------------------------------
        # 外部 provider source ingestion 新鮮度
        # -----------------------------------------------------------------
        - alert: SourceProviderIngestionStale
          expr: |
            time() - max by (source) (
              awoooi_alert_chain_last_success_timestamp{source=~"sentry|signoz"}
            ) > 86400
          for: 15m
          labels:
            severity: warning
            service: alert-chain
            component: source-ingestion
            team: platform
          annotations:
            summary: "{{ $labels.source }} source ingestion 超過 24 小時未更新"
            description: "{{ $labels.source }} webhook endpoint 可能仍健康，但 AwoooP source dossier 已超過 24 小時沒有新事件。這是 provider ingestion / upstream smoke / correlation freshness 缺口，不是 Alertmanager 主鏈路故障。"
            runbook_url: "https://awoooi.internal/runbooks/alert-chain"

        # -----------------------------------------------------------------
        # 告警鏈路健康狀態
        # -----------------------------------------------------------------
        - alert: AlertChainUnhealthy
          expr: |
            awoooi_alert_chain_healthy == 0
          for: 5m
          labels:
            severity: critical
            service: alert-chain
            team: platform
          annotations:
            summary: "告警鏈路不健康"
            description: "{{ $labels.source }} 告警鏈路標記為不健康，最近處理失敗"

    # =========================================================================
    # Telegram 通知監控
    # =========================================================================
    - name: telegram_notifications
      rules:
        - alert: TelegramNotificationsFailing
          expr: |
            sum(rate(awoooi_telegram_notifications_total{
              status="failed"
            }[15m])) / sum(rate(awoooi_telegram_notifications_total[15m])) > 0.3
          for: 15m
          labels:
            severity: warning
            service: telegram
            team: platform
          annotations:
            summary: "Telegram 通知失敗率 > 30%"
            description: "告警可能無法正確發送到 Telegram"

    # =========================================================================
    # 異常頻率升級監控 (ADR-037)
    # =========================================================================
    - name: anomaly_escalation
      rules:
        - alert: FrequentAnomalyEscalation
          expr: |
            sum(rate(awoooi_anomaly_escalation_total{
              level=~"ESCALATE|PERMANENT_FIX"
            }[1h])) > 5
          for: 5m
          labels:
            severity: warning
            service: anomaly-counter
            team: backend
          annotations:
            summary: "頻繁異常升級告警"
            description: "過去 1 小時有超過 5 次異常升級到 ESCALATE 或 PERMANENT_FIX 級別"

        - alert: PermanentFixRequired
          expr: |
            sum(rate(awoooi_anomaly_escalation_total{
              level="PERMANENT_FIX"
            }[1h])) > 0
          for: 1m
          labels:
            severity: critical
            service: anomaly-counter
            team: backend
          annotations:
            summary: "需要永久修復的異常"
            description: "有異常升級到 PERMANENT_FIX 級別，需要根本修復而非重啟"

    # =========================================================================
    # 自動修復監控
    # =========================================================================
    - name: auto_repair
      rules:
        - alert: AutoRepairLowSuccessRate
          expr: |
            awoooi_auto_repair_success_rate < 0.3
          for: 30m
          labels:
            severity: warning
            service: auto-repair
            team: backend
          annotations:
            summary: "自動修復成功率過低 (< 30%)"
            description: "動作 {{ $labels.action }} 的成功率只有 {{ $value | humanizePercentage }}，建議檢查修復邏輯"
