awoooi/apps/api/src/services/alert_chain_metrics_service.py

"""DB-backed Alert Chain metric refresh.

`record_alert_chain_success()` updates Prometheus gauges in process memory. A
deployment or Pod restart clears that memory, so the first scrape after deploy
can report `awoooi_alert_chain_last_success_timestamp` as missing even though
Alertmanager / Sentry / SigNoz evidence is already in DB.
"""

from __future__ import annotations

from dataclasses import dataclass
from time import time
from typing import Any, Mapping

import structlog
from sqlalchemy import text

from src.core.metrics import ALERT_CHAIN_LAST_SUCCESS
from src.db.base import get_db_context

logger = structlog.get_logger(__name__)

_REFRESH_INTERVAL_SECONDS = 15


@dataclass(frozen=True)
class AlertChainEvidenceSample:
    source: str
    last_success_timestamp: float
    evidence_source: str


class AlertChainMetricsService:
    """Refresh process-local alert-chain gauges from durable DB evidence."""

    def __init__(self) -> None:
        self._last_refresh_at = 0.0
        self._last_samples: list[AlertChainEvidenceSample] = []

    async def refresh_last_success_gauge(
        self,
        *,
        project_id: str = "awoooi",
        force: bool = False,
    ) -> list[AlertChainEvidenceSample]:
        now_ts = time()
        if (
            not force
            and self._last_samples
            and now_ts - self._last_refresh_at < _REFRESH_INTERVAL_SECONDS
        ):
            apply_alert_chain_last_success_samples(self._last_samples)
            return self._last_samples

        samples = await self.fetch_latest_success_samples(project_id=project_id)
        apply_alert_chain_last_success_samples(samples)
        self._last_samples = samples
        self._last_refresh_at = now_ts
        return samples

    async def fetch_latest_success_samples(
        self,
        *,
        project_id: str = "awoooi",
    ) -> list[AlertChainEvidenceSample]:
        async with get_db_context(project_id) as db:
            result = await db.execute(text(_ALERT_CHAIN_EVIDENCE_SQL), {"project_id": project_id})
            return merge_alert_chain_evidence_rows(result.mappings().all())


def apply_alert_chain_last_success_samples(samples: list[AlertChainEvidenceSample]) -> None:
    """Write durable last-success timestamps into the existing Prometheus gauge.

    Do not write `awoooi_alert_chain_healthy` here: that gauge represents an
    explicit runtime failure signal. Durable success evidence should not mask a
    later in-memory failure from the same process.
    """
    for sample in samples:
        ALERT_CHAIN_LAST_SUCCESS.labels(source=sample.source).set(
            sample.last_success_timestamp
        )


def merge_alert_chain_evidence_rows(
    rows: list[Mapping[str, Any]] | list[Any],
) -> list[AlertChainEvidenceSample]:
    """Normalize SQL rows and keep the newest evidence per source."""
    latest_by_source: dict[str, AlertChainEvidenceSample] = {}

    for row in rows:
        source = str(_row_value(row, "source") or "").strip().lower()
        evidence_source = str(_row_value(row, "evidence_source") or "db").strip()
        raw_timestamp = _row_value(row, "last_success_timestamp")
        if not source or raw_timestamp is None:
            continue

        try:
            timestamp = float(raw_timestamp)
        except (TypeError, ValueError):
            continue
        if timestamp <= 0:
            continue

        sample = AlertChainEvidenceSample(
            source=source,
            last_success_timestamp=timestamp,
            evidence_source=evidence_source,
        )
        previous = latest_by_source.get(source)
        if previous is None or sample.last_success_timestamp > previous.last_success_timestamp:
            latest_by_source[source] = sample

    return sorted(latest_by_source.values(), key=lambda item: item.source)


def _row_value(row: Mapping[str, Any] | Any, key: str) -> Any:
    if isinstance(row, Mapping):
        return row.get(key)
    return getattr(row, key, None)


_ALERT_CHAIN_EVIDENCE_SQL = """
    WITH conversation_evidence AS (
        SELECT
            lower(platform_subject_id) AS source,
            EXTRACT(EPOCH FROM MAX(COALESCE(provider_ts, received_at))) AS last_success_timestamp,
            'awooop_conversation_event' AS evidence_source
        FROM awooop_conversation_event
        WHERE project_id = :project_id
          AND channel_type = 'internal'
          AND lower(COALESCE(platform_subject_id, '')) IN ('alertmanager', 'sentry', 'signoz')
        GROUP BY lower(platform_subject_id)
    ),
    alertmanager_log_evidence AS (
        SELECT
            'alertmanager' AS source,
            EXTRACT(EPOCH FROM MAX(created_at)) AS last_success_timestamp,
            'alert_operation_log' AS evidence_source
        FROM alert_operation_log
        WHERE event_type = 'ALERT_RECEIVED'
          AND actor = 'alertmanager'
    )
    SELECT source, last_success_timestamp, evidence_source
    FROM conversation_evidence
    WHERE last_success_timestamp IS NOT NULL
    UNION ALL
    SELECT source, last_success_timestamp, evidence_source
    FROM alertmanager_log_evidence
    WHERE last_success_timestamp IS NOT NULL
"""


_alert_chain_metrics_service: AlertChainMetricsService | None = None


def get_alert_chain_metrics_service() -> AlertChainMetricsService:
    global _alert_chain_metrics_service
    if _alert_chain_metrics_service is None:
        _alert_chain_metrics_service = AlertChainMetricsService()
    return _alert_chain_metrics_service