from __future__ import annotations from datetime import UTC, datetime, timedelta from unittest.mock import AsyncMock import pytest from fastapi import BackgroundTasks from starlette.requests import Request from src.api.v1 import webhooks as webhooks_module from src.api.v1.webhooks import ( AlertmanagerAlert, AlertmanagerPayload, _normalized_alertmanager_resolved_identity, alertmanager_webhook, ) from src.services.alertmanager_recovery_lifecycle import ( AlertmanagerRecoveryResult, _merge_reservation_observation, _outcome_matches_current_cycle, _reservation_decision, handle_alertmanager_resolved, send_alertmanager_recovery, ) from src.services.telegram_gateway import _telegram_destination_binding def _request() -> Request: return Request( { "type": "http", "method": "POST", "path": "/api/v1/webhooks/alertmanager", "scheme": "http", "server": ("testserver", 80), "client": ("127.0.0.1", 50000), "query_string": b"", "headers": [], } ) def test_reservation_suppresses_acknowledged_and_fresh_inflight_duplicates() -> None: now = datetime.now(UTC) acknowledged = _reservation_decision( { "status": "recovered_acknowledged", "delivery_attempts": 1, }, incident_id="INC-20260722-ABC123", now=now, ) inflight = _reservation_decision( { "status": "recovery_pending_delivery", "delivery_attempts": 1, "reserved_at": (now - timedelta(seconds=30)).isoformat(), }, incident_id="INC-20260722-ABC123", now=now, ) assert acknowledged == { "admitted": False, "status": "duplicate_recovered_suppressed", "attempts": 1, } assert inflight == { "admitted": False, "status": "duplicate_recovery_inflight_suppressed", "attempts": 1, } def test_reservation_allows_bounded_retry_but_fails_closed_without_identity() -> None: now = datetime.now(UTC) retry = _reservation_decision( { "status": "recovery_delivery_failed", "delivery_attempts": 1, }, incident_id="INC-20260722-ABC123", now=now, ) exhausted = _reservation_decision( { "status": "recovery_delivery_failed", "delivery_attempts": 3, }, incident_id="INC-20260722-ABC123", now=now, ) unmatched = _reservation_decision( None, incident_id="", now=now, ) assert retry == { "admitted": True, "status": "recovery_pending_delivery", "attempts": 2, } assert exhausted["status"] == "recovery_delivery_retry_exhausted" assert exhausted["admitted"] is False assert unmatched["status"] == "resolved_identity_unmatched" assert unmatched["admitted"] is False def test_duplicate_observation_preserves_terminal_ack_and_inflight_reservation() -> ( None ): now = datetime.now(UTC) acknowledged = { "status": "recovered_acknowledged", "delivery_attempts": 1, "reserved_at": (now - timedelta(seconds=60)).isoformat(), "provider_send_performed": True, "provider_acknowledged": True, "delivery_ack": {"provider_message_id": "991"}, } decision = _reservation_decision( acknowledged, incident_id="INC-20260722-ABC123", now=now, ) updated = _merge_reservation_observation( acknowledged, decision=decision, observed={"source_ends_at": "2026-07-22T02:00:00Z"}, now=now, is_new=False, ) assert updated["status"] == "recovered_acknowledged" assert updated["reserved_at"] == acknowledged["reserved_at"] assert updated["provider_acknowledged"] is True assert updated["delivery_ack"] == {"provider_message_id": "991"} assert updated["last_duplicate_status"] == "duplicate_recovered_suppressed" inflight = { "status": "recovery_pending_delivery", "delivery_attempts": 1, "reserved_at": (now - timedelta(seconds=30)).isoformat(), "provider_send_performed": False, "provider_acknowledged": False, } inflight_decision = _reservation_decision( inflight, incident_id="INC-20260722-ABC123", now=now, ) inflight_updated = _merge_reservation_observation( inflight, decision=inflight_decision, observed={"source_ends_at": "2026-07-22T02:00:00Z"}, now=now, is_new=False, ) assert inflight_updated["status"] == "recovery_pending_delivery" assert inflight_updated["reserved_at"] == inflight["reserved_at"] assert inflight_updated["last_duplicate_status"] == ( "duplicate_recovery_inflight_suppressed" ) def test_provider_send_without_destination_ack_is_never_resent_automatically() -> None: decision = _reservation_decision( { "status": "recovery_delivery_failed", "delivery_attempts": 1, "provider_send_performed": True, "provider_acknowledged": False, }, incident_id="INC-20260722-ABC123", now=datetime.now(UTC), ) assert decision == { "admitted": False, "status": "recovery_delivery_ack_unresolved", "attempts": 1, } def test_new_incident_reopens_same_fingerprint_without_losing_prior_ack() -> None: now = datetime.now(UTC) previous = { "incident_id": "INC-20260721-OLD001", "approval_id": "approval-old", "status": "recovered_acknowledged", "delivery_attempts": 1, "provider_send_performed": True, "provider_acknowledged": True, "delivery_ack": {"provider_message_id": "880"}, "completed_at": (now - timedelta(days=1)).isoformat(), } decision = _reservation_decision( previous, incident_id="INC-20260722-NEW001", now=now, ) updated = _merge_reservation_observation( previous, decision=decision, observed={ "incident_id": "INC-20260722-NEW001", "approval_id": "approval-new", }, now=now, is_new=False, ) assert decision["new_incident_cycle"] is True assert updated["incident_id"] == "INC-20260722-NEW001" assert updated["status"] == "recovery_pending_delivery" assert updated["delivery_attempts"] == 1 assert updated["provider_acknowledged"] is False assert "completed_at" not in updated assert updated["completed_cycles"][-1]["incident_id"] == ("INC-20260721-OLD001") assert updated["completed_cycles"][-1]["provider_acknowledged"] is True def test_outcome_state_is_monotonic_and_cycle_bound() -> None: lifecycle = { "incident_id": "INC-20260722-ABC123", "status": "recovery_pending_delivery", "delivery_attempts": 2, } assert not _outcome_matches_current_cycle( lifecycle, incident_id="INC-20260721-OLD001", delivery_attempt=1, acknowledged=True, ) assert not _outcome_matches_current_cycle( lifecycle, incident_id="INC-20260722-ABC123", delivery_attempt=1, acknowledged=False, ) assert _outcome_matches_current_cycle( lifecycle, incident_id="INC-20260722-ABC123", delivery_attempt=1, acknowledged=True, ) assert not _outcome_matches_current_cycle( {**lifecycle, "status": "recovered_acknowledged"}, incident_id="INC-20260722-ABC123", delivery_attempt=2, acknowledged=False, ) @pytest.mark.asyncio async def test_resolved_handler_updates_incident_and_records_destination_ack() -> None: correlation_lookup = AsyncMock( return_value={ "approval_id": "approval-1", "incident_id": "INC-20260722-ABC123", "telegram_message_id": 881, } ) reservation = AsyncMock( return_value={ "event_id": "0b7d5caf-4aa1-4e5a-8f51-987d66db7c20", "admitted": True, "status": "recovery_pending_delivery", "duplicate": False, "delivery_attempt": 1, } ) incident_resolver = AsyncMock(return_value=True) sender = AsyncMock( return_value={ "acknowledged": True, "provider_send_performed": True, "ack": { "provider_message_id": "991", "destination_binding": "binding-1", "provider_destination_binding": "binding-1", }, } ) outcome_recorder = AsyncMock(return_value=None) result = await handle_alertmanager_resolved( fingerprint="a" * 32, alertname="DockerContainerUnhealthy", severity="critical", namespace="default", target_resource="alertmanager", source_alert_id="source-fp-1", source_ends_at="2026-07-22T02:00:00Z", correlation_lookup=correlation_lookup, recovery_reservation=reservation, incident_resolver=incident_resolver, recovery_sender=sender, outcome_recorder=outcome_recorder, ) assert result.status == "recovered_acknowledged" assert result.provider_acknowledged is True incident_resolver.assert_awaited_once_with("INC-20260722-ABC123") sender.assert_awaited_once() assert sender.await_args.kwargs["reply_to_message_id"] == 881 outcome_recorder.assert_awaited_once() assert outcome_recorder.await_args.kwargs["delivery"]["acknowledged"] is True @pytest.mark.asyncio async def test_delivery_retry_uses_identity_from_durable_lifecycle_receipt() -> None: correlation_lookup = AsyncMock(return_value=None) reservation = AsyncMock( return_value={ "event_id": "0b7d5caf-4aa1-4e5a-8f51-987d66db7c20", "admitted": True, "status": "recovery_pending_delivery", "duplicate": False, "delivery_attempt": 2, "incident_id": "INC-20260722-ABC123", "telegram_message_id": 881, } ) incident_resolver = AsyncMock(return_value=True) sender = AsyncMock( return_value={ "acknowledged": True, "provider_send_performed": True, "ack": {"provider_message_id": "992"}, } ) outcome_recorder = AsyncMock(return_value=None) result = await handle_alertmanager_resolved( fingerprint="a" * 32, alertname="DockerContainerUnhealthy", severity="critical", namespace="default", target_resource="alertmanager", source_alert_id="source-fp-1", source_ends_at="2026-07-22T02:00:00Z", correlation_lookup=correlation_lookup, recovery_reservation=reservation, incident_resolver=incident_resolver, recovery_sender=sender, outcome_recorder=outcome_recorder, ) assert result.status == "recovered_acknowledged" assert result.incident_id == "INC-20260722-ABC123" incident_resolver.assert_awaited_once_with("INC-20260722-ABC123") assert sender.await_args.kwargs["reply_to_message_id"] == 881 assert outcome_recorder.await_args.kwargs["delivery_attempt"] == 2 @pytest.mark.asyncio async def test_duplicate_resolved_event_never_resolves_or_sends_again() -> None: correlation_lookup = AsyncMock( return_value={ "approval_id": "approval-1", "incident_id": "INC-20260722-ABC123", "telegram_message_id": 881, } ) reservation = AsyncMock( return_value={ "event_id": "0b7d5caf-4aa1-4e5a-8f51-987d66db7c20", "admitted": False, "status": "duplicate_recovered_suppressed", "duplicate": True, "delivery_attempt": 1, "provider_acknowledged": True, "provider_send_performed": True, } ) incident_resolver = AsyncMock( side_effect=AssertionError("duplicate must not resolve incident again") ) sender = AsyncMock( side_effect=AssertionError("duplicate must not send Telegram again") ) outcome_recorder = AsyncMock( side_effect=AssertionError("duplicate must not rewrite acknowledged outcome") ) result = await handle_alertmanager_resolved( fingerprint="a" * 32, alertname="DockerContainerUnhealthy", severity="critical", namespace="default", target_resource="alertmanager", source_alert_id="source-fp-1", source_ends_at="2026-07-22T02:00:00Z", correlation_lookup=correlation_lookup, recovery_reservation=reservation, incident_resolver=incident_resolver, recovery_sender=sender, outcome_recorder=outcome_recorder, ) assert result.status == "duplicate_recovered_suppressed" assert result.duplicate is True assert result.provider_acknowledged is True assert result.provider_send_performed is True incident_resolver.assert_not_awaited() sender.assert_not_awaited() outcome_recorder.assert_not_awaited() @pytest.mark.asyncio async def test_unmatched_resolved_identity_records_no_false_recovery() -> None: correlation_lookup = AsyncMock(return_value=None) reservation = AsyncMock( return_value={ "event_id": "0b7d5caf-4aa1-4e5a-8f51-987d66db7c20", "admitted": False, "status": "resolved_identity_unmatched", "duplicate": True, "delivery_attempt": 0, } ) incident_resolver = AsyncMock( side_effect=AssertionError("unmatched identity must remain fail closed") ) sender = AsyncMock( side_effect=AssertionError("unmatched identity must not send recovery") ) result = await handle_alertmanager_resolved( fingerprint="b" * 32, alertname="UnknownAlert", severity="warning", namespace="default", target_resource="unknown-target", source_alert_id="source-fp-2", source_ends_at="2026-07-22T02:00:00Z", correlation_lookup=correlation_lookup, recovery_reservation=reservation, incident_resolver=incident_resolver, recovery_sender=sender, outcome_recorder=AsyncMock(), ) assert result.status == "resolved_identity_unmatched" assert result.provider_acknowledged is False incident_resolver.assert_not_awaited() sender.assert_not_awaited() @pytest.mark.asyncio async def test_resolved_webhook_enters_canonical_lifecycle( monkeypatch: pytest.MonkeyPatch, ) -> None: lifecycle = AsyncMock( return_value=AlertmanagerRecoveryResult( status="recovered_acknowledged", fingerprint="a" * 32, incident_id="INC-20260722-ABC123", lifecycle_event_id="0b7d5caf-4aa1-4e5a-8f51-987d66db7c20", duplicate=False, provider_acknowledged=True, provider_send_performed=True, ) ) monkeypatch.setattr( webhooks_module, "handle_alertmanager_resolved", lifecycle, ) alert = AlertmanagerAlert( status="resolved", labels={ "alertname": "DockerContainerUnhealthy", "severity": "critical", "namespace": "default", "name": "alertmanager", }, annotations={"summary": "container unhealthy"}, startsAt="2026-07-22T01:00:00Z", endsAt="2026-07-22T02:00:00Z", fingerprint="provider-fingerprint", ) identity = _normalized_alertmanager_resolved_identity(alert) response = await alertmanager_webhook( _request(), AlertmanagerPayload(status="resolved", alerts=[alert]), BackgroundTasks(), ) assert response.success is True assert response.alert_id == "provider-fingerprint" assert "provider_ack=true" in response.message lifecycle.assert_awaited_once_with( fingerprint=identity["fingerprint"], alertname="DockerContainerUnhealthy", severity="critical", namespace="default", target_resource="alertmanager", source_alert_id="provider-fingerprint", source_ends_at="2026-07-22T02:00:00Z", ) @pytest.mark.asyncio async def test_recovery_sender_requires_destination_bound_provider_receipt( monkeypatch: pytest.MonkeyPatch, ) -> None: from src.services import telegram_gateway as gateway_module destination_binding = _telegram_destination_binding(-100123) provider_result = { "ok": True, "result": { "message_id": 991, "chat": {"id": -100123}, }, "_awooop_delivery_status": "sent", "_awooop_provider_send_performed": True, "_awoooi_canonical_route_receipt": { "schema_version": "telegram_canonical_egress_receipt_v1", "decision": "allowed", "provider_send_performed": True, "sender_bot_alias": "tsenyang_bot", "destination_alias": "awoooi_sre_war_room", "destination_binding": destination_binding, }, "_awoooi_delivery_context": { "schema_version": "telegram_delivery_context_v1", "sender_bot_alias": "tsenyang_bot", "destination_alias": "awoooi_sre_war_room", "destination_binding": destination_binding, "payload_destination_binding": destination_binding, "provider_destination_binding": destination_binding, "provider_destination_verification_method": ( "requested_chat_id_matches_provider_chat_id" ), "destination_binding_verified": True, }, } class _Gateway: send_canonical_message = AsyncMock(return_value=provider_result) gateway = _Gateway() monkeypatch.setattr(gateway_module, "get_telegram_gateway", lambda: gateway) delivery = await send_alertmanager_recovery( incident_id="INC-20260722-ABC123", alertname="DockerContainerUnhealthy", severity="critical", target_resource="alertmanager", fingerprint="a" * 32, reply_to_message_id=881, ) assert delivery["acknowledged"] is True assert delivery["provider_send_performed"] is True assert delivery["ack"]["provider_message_id"] == "991" assert delivery["ack"]["destination_binding"] == destination_binding assert delivery["ack"]["provider_destination_binding"] == destination_binding assert gateway.send_canonical_message.await_args.kwargs == { "product_id": "awoooi", "signal_family": "incident_lifecycle", "severity": "P0", "text": gateway.send_canonical_message.await_args.kwargs["text"], "reply_to_message_id": 881, } def test_resolved_normalization_is_stable_for_same_alert_identity() -> None: common = { "labels": { "alertname": "DockerContainerUnhealthy", "severity": "critical", "namespace": "default", "name": "alertmanager", }, "annotations": {"summary": "container unhealthy"}, "startsAt": "2026-07-22T01:00:00Z", "fingerprint": "provider-fingerprint", } firing = _normalized_alertmanager_resolved_identity( AlertmanagerAlert(status="firing", **common) ) resolved = _normalized_alertmanager_resolved_identity( AlertmanagerAlert( status="resolved", endsAt="2026-07-22T02:00:00Z", **common, ) ) assert resolved["fingerprint"] == firing["fingerprint"] assert resolved["target_resource"] == "alertmanager" assert resolved["namespace"] == "default"