604 lines
19 KiB
Python
604 lines
19 KiB
Python
from __future__ import annotations
|
|
|
|
from datetime import UTC, datetime, timedelta
|
|
from unittest.mock import AsyncMock
|
|
|
|
import pytest
|
|
from fastapi import BackgroundTasks
|
|
from starlette.requests import Request
|
|
|
|
from src.api.v1 import webhooks as webhooks_module
|
|
from src.api.v1.webhooks import (
|
|
AlertmanagerAlert,
|
|
AlertmanagerPayload,
|
|
_normalized_alertmanager_resolved_identity,
|
|
alertmanager_webhook,
|
|
)
|
|
from src.services.alertmanager_recovery_lifecycle import (
|
|
AlertmanagerRecoveryResult,
|
|
_merge_reservation_observation,
|
|
_outcome_matches_current_cycle,
|
|
_reservation_decision,
|
|
handle_alertmanager_resolved,
|
|
send_alertmanager_recovery,
|
|
)
|
|
from src.services.telegram_gateway import _telegram_destination_binding
|
|
|
|
|
|
def _request() -> Request:
|
|
return Request(
|
|
{
|
|
"type": "http",
|
|
"method": "POST",
|
|
"path": "/api/v1/webhooks/alertmanager",
|
|
"scheme": "http",
|
|
"server": ("testserver", 80),
|
|
"client": ("127.0.0.1", 50000),
|
|
"query_string": b"",
|
|
"headers": [],
|
|
}
|
|
)
|
|
|
|
|
|
def test_reservation_suppresses_acknowledged_and_fresh_inflight_duplicates() -> None:
|
|
now = datetime.now(UTC)
|
|
|
|
acknowledged = _reservation_decision(
|
|
{
|
|
"status": "recovered_acknowledged",
|
|
"delivery_attempts": 1,
|
|
},
|
|
incident_id="INC-20260722-ABC123",
|
|
now=now,
|
|
)
|
|
inflight = _reservation_decision(
|
|
{
|
|
"status": "recovery_pending_delivery",
|
|
"delivery_attempts": 1,
|
|
"reserved_at": (now - timedelta(seconds=30)).isoformat(),
|
|
},
|
|
incident_id="INC-20260722-ABC123",
|
|
now=now,
|
|
)
|
|
|
|
assert acknowledged == {
|
|
"admitted": False,
|
|
"status": "duplicate_recovered_suppressed",
|
|
"attempts": 1,
|
|
}
|
|
assert inflight == {
|
|
"admitted": False,
|
|
"status": "duplicate_recovery_inflight_suppressed",
|
|
"attempts": 1,
|
|
}
|
|
|
|
|
|
def test_reservation_allows_bounded_retry_but_fails_closed_without_identity() -> None:
|
|
now = datetime.now(UTC)
|
|
retry = _reservation_decision(
|
|
{
|
|
"status": "recovery_delivery_failed",
|
|
"delivery_attempts": 1,
|
|
},
|
|
incident_id="INC-20260722-ABC123",
|
|
now=now,
|
|
)
|
|
exhausted = _reservation_decision(
|
|
{
|
|
"status": "recovery_delivery_failed",
|
|
"delivery_attempts": 3,
|
|
},
|
|
incident_id="INC-20260722-ABC123",
|
|
now=now,
|
|
)
|
|
unmatched = _reservation_decision(
|
|
None,
|
|
incident_id="",
|
|
now=now,
|
|
)
|
|
|
|
assert retry == {
|
|
"admitted": True,
|
|
"status": "recovery_pending_delivery",
|
|
"attempts": 2,
|
|
}
|
|
assert exhausted["status"] == "recovery_delivery_retry_exhausted"
|
|
assert exhausted["admitted"] is False
|
|
assert unmatched["status"] == "resolved_identity_unmatched"
|
|
assert unmatched["admitted"] is False
|
|
|
|
|
|
def test_duplicate_observation_preserves_terminal_ack_and_inflight_reservation() -> (
|
|
None
|
|
):
|
|
now = datetime.now(UTC)
|
|
acknowledged = {
|
|
"status": "recovered_acknowledged",
|
|
"delivery_attempts": 1,
|
|
"reserved_at": (now - timedelta(seconds=60)).isoformat(),
|
|
"provider_send_performed": True,
|
|
"provider_acknowledged": True,
|
|
"delivery_ack": {"provider_message_id": "991"},
|
|
}
|
|
decision = _reservation_decision(
|
|
acknowledged,
|
|
incident_id="INC-20260722-ABC123",
|
|
now=now,
|
|
)
|
|
|
|
updated = _merge_reservation_observation(
|
|
acknowledged,
|
|
decision=decision,
|
|
observed={"source_ends_at": "2026-07-22T02:00:00Z"},
|
|
now=now,
|
|
is_new=False,
|
|
)
|
|
|
|
assert updated["status"] == "recovered_acknowledged"
|
|
assert updated["reserved_at"] == acknowledged["reserved_at"]
|
|
assert updated["provider_acknowledged"] is True
|
|
assert updated["delivery_ack"] == {"provider_message_id": "991"}
|
|
assert updated["last_duplicate_status"] == "duplicate_recovered_suppressed"
|
|
|
|
inflight = {
|
|
"status": "recovery_pending_delivery",
|
|
"delivery_attempts": 1,
|
|
"reserved_at": (now - timedelta(seconds=30)).isoformat(),
|
|
"provider_send_performed": False,
|
|
"provider_acknowledged": False,
|
|
}
|
|
inflight_decision = _reservation_decision(
|
|
inflight,
|
|
incident_id="INC-20260722-ABC123",
|
|
now=now,
|
|
)
|
|
inflight_updated = _merge_reservation_observation(
|
|
inflight,
|
|
decision=inflight_decision,
|
|
observed={"source_ends_at": "2026-07-22T02:00:00Z"},
|
|
now=now,
|
|
is_new=False,
|
|
)
|
|
|
|
assert inflight_updated["status"] == "recovery_pending_delivery"
|
|
assert inflight_updated["reserved_at"] == inflight["reserved_at"]
|
|
assert inflight_updated["last_duplicate_status"] == (
|
|
"duplicate_recovery_inflight_suppressed"
|
|
)
|
|
|
|
|
|
def test_provider_send_without_destination_ack_is_never_resent_automatically() -> None:
|
|
decision = _reservation_decision(
|
|
{
|
|
"status": "recovery_delivery_failed",
|
|
"delivery_attempts": 1,
|
|
"provider_send_performed": True,
|
|
"provider_acknowledged": False,
|
|
},
|
|
incident_id="INC-20260722-ABC123",
|
|
now=datetime.now(UTC),
|
|
)
|
|
|
|
assert decision == {
|
|
"admitted": False,
|
|
"status": "recovery_delivery_ack_unresolved",
|
|
"attempts": 1,
|
|
}
|
|
|
|
|
|
def test_new_incident_reopens_same_fingerprint_without_losing_prior_ack() -> None:
|
|
now = datetime.now(UTC)
|
|
previous = {
|
|
"incident_id": "INC-20260721-OLD001",
|
|
"approval_id": "approval-old",
|
|
"status": "recovered_acknowledged",
|
|
"delivery_attempts": 1,
|
|
"provider_send_performed": True,
|
|
"provider_acknowledged": True,
|
|
"delivery_ack": {"provider_message_id": "880"},
|
|
"completed_at": (now - timedelta(days=1)).isoformat(),
|
|
}
|
|
|
|
decision = _reservation_decision(
|
|
previous,
|
|
incident_id="INC-20260722-NEW001",
|
|
now=now,
|
|
)
|
|
updated = _merge_reservation_observation(
|
|
previous,
|
|
decision=decision,
|
|
observed={
|
|
"incident_id": "INC-20260722-NEW001",
|
|
"approval_id": "approval-new",
|
|
},
|
|
now=now,
|
|
is_new=False,
|
|
)
|
|
|
|
assert decision["new_incident_cycle"] is True
|
|
assert updated["incident_id"] == "INC-20260722-NEW001"
|
|
assert updated["status"] == "recovery_pending_delivery"
|
|
assert updated["delivery_attempts"] == 1
|
|
assert updated["provider_acknowledged"] is False
|
|
assert "completed_at" not in updated
|
|
assert updated["completed_cycles"][-1]["incident_id"] == ("INC-20260721-OLD001")
|
|
assert updated["completed_cycles"][-1]["provider_acknowledged"] is True
|
|
|
|
|
|
def test_outcome_state_is_monotonic_and_cycle_bound() -> None:
|
|
lifecycle = {
|
|
"incident_id": "INC-20260722-ABC123",
|
|
"status": "recovery_pending_delivery",
|
|
"delivery_attempts": 2,
|
|
}
|
|
|
|
assert not _outcome_matches_current_cycle(
|
|
lifecycle,
|
|
incident_id="INC-20260721-OLD001",
|
|
delivery_attempt=1,
|
|
acknowledged=True,
|
|
)
|
|
assert not _outcome_matches_current_cycle(
|
|
lifecycle,
|
|
incident_id="INC-20260722-ABC123",
|
|
delivery_attempt=1,
|
|
acknowledged=False,
|
|
)
|
|
assert _outcome_matches_current_cycle(
|
|
lifecycle,
|
|
incident_id="INC-20260722-ABC123",
|
|
delivery_attempt=1,
|
|
acknowledged=True,
|
|
)
|
|
assert not _outcome_matches_current_cycle(
|
|
{**lifecycle, "status": "recovered_acknowledged"},
|
|
incident_id="INC-20260722-ABC123",
|
|
delivery_attempt=2,
|
|
acknowledged=False,
|
|
)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_resolved_handler_updates_incident_and_records_destination_ack() -> None:
|
|
correlation_lookup = AsyncMock(
|
|
return_value={
|
|
"approval_id": "approval-1",
|
|
"incident_id": "INC-20260722-ABC123",
|
|
"telegram_message_id": 881,
|
|
}
|
|
)
|
|
reservation = AsyncMock(
|
|
return_value={
|
|
"event_id": "0b7d5caf-4aa1-4e5a-8f51-987d66db7c20",
|
|
"admitted": True,
|
|
"status": "recovery_pending_delivery",
|
|
"duplicate": False,
|
|
"delivery_attempt": 1,
|
|
}
|
|
)
|
|
incident_resolver = AsyncMock(return_value=True)
|
|
sender = AsyncMock(
|
|
return_value={
|
|
"acknowledged": True,
|
|
"provider_send_performed": True,
|
|
"ack": {
|
|
"provider_message_id": "991",
|
|
"destination_binding": "binding-1",
|
|
"provider_destination_binding": "binding-1",
|
|
},
|
|
}
|
|
)
|
|
outcome_recorder = AsyncMock(return_value=None)
|
|
|
|
result = await handle_alertmanager_resolved(
|
|
fingerprint="a" * 32,
|
|
alertname="DockerContainerUnhealthy",
|
|
severity="critical",
|
|
namespace="default",
|
|
target_resource="alertmanager",
|
|
source_alert_id="source-fp-1",
|
|
source_ends_at="2026-07-22T02:00:00Z",
|
|
correlation_lookup=correlation_lookup,
|
|
recovery_reservation=reservation,
|
|
incident_resolver=incident_resolver,
|
|
recovery_sender=sender,
|
|
outcome_recorder=outcome_recorder,
|
|
)
|
|
|
|
assert result.status == "recovered_acknowledged"
|
|
assert result.provider_acknowledged is True
|
|
incident_resolver.assert_awaited_once_with("INC-20260722-ABC123")
|
|
sender.assert_awaited_once()
|
|
assert sender.await_args.kwargs["reply_to_message_id"] == 881
|
|
outcome_recorder.assert_awaited_once()
|
|
assert outcome_recorder.await_args.kwargs["delivery"]["acknowledged"] is True
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_delivery_retry_uses_identity_from_durable_lifecycle_receipt() -> None:
|
|
correlation_lookup = AsyncMock(return_value=None)
|
|
reservation = AsyncMock(
|
|
return_value={
|
|
"event_id": "0b7d5caf-4aa1-4e5a-8f51-987d66db7c20",
|
|
"admitted": True,
|
|
"status": "recovery_pending_delivery",
|
|
"duplicate": False,
|
|
"delivery_attempt": 2,
|
|
"incident_id": "INC-20260722-ABC123",
|
|
"telegram_message_id": 881,
|
|
}
|
|
)
|
|
incident_resolver = AsyncMock(return_value=True)
|
|
sender = AsyncMock(
|
|
return_value={
|
|
"acknowledged": True,
|
|
"provider_send_performed": True,
|
|
"ack": {"provider_message_id": "992"},
|
|
}
|
|
)
|
|
outcome_recorder = AsyncMock(return_value=None)
|
|
|
|
result = await handle_alertmanager_resolved(
|
|
fingerprint="a" * 32,
|
|
alertname="DockerContainerUnhealthy",
|
|
severity="critical",
|
|
namespace="default",
|
|
target_resource="alertmanager",
|
|
source_alert_id="source-fp-1",
|
|
source_ends_at="2026-07-22T02:00:00Z",
|
|
correlation_lookup=correlation_lookup,
|
|
recovery_reservation=reservation,
|
|
incident_resolver=incident_resolver,
|
|
recovery_sender=sender,
|
|
outcome_recorder=outcome_recorder,
|
|
)
|
|
|
|
assert result.status == "recovered_acknowledged"
|
|
assert result.incident_id == "INC-20260722-ABC123"
|
|
incident_resolver.assert_awaited_once_with("INC-20260722-ABC123")
|
|
assert sender.await_args.kwargs["reply_to_message_id"] == 881
|
|
assert outcome_recorder.await_args.kwargs["delivery_attempt"] == 2
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_duplicate_resolved_event_never_resolves_or_sends_again() -> None:
|
|
correlation_lookup = AsyncMock(
|
|
return_value={
|
|
"approval_id": "approval-1",
|
|
"incident_id": "INC-20260722-ABC123",
|
|
"telegram_message_id": 881,
|
|
}
|
|
)
|
|
reservation = AsyncMock(
|
|
return_value={
|
|
"event_id": "0b7d5caf-4aa1-4e5a-8f51-987d66db7c20",
|
|
"admitted": False,
|
|
"status": "duplicate_recovered_suppressed",
|
|
"duplicate": True,
|
|
"delivery_attempt": 1,
|
|
"provider_acknowledged": True,
|
|
"provider_send_performed": True,
|
|
}
|
|
)
|
|
incident_resolver = AsyncMock(
|
|
side_effect=AssertionError("duplicate must not resolve incident again")
|
|
)
|
|
sender = AsyncMock(
|
|
side_effect=AssertionError("duplicate must not send Telegram again")
|
|
)
|
|
outcome_recorder = AsyncMock(
|
|
side_effect=AssertionError("duplicate must not rewrite acknowledged outcome")
|
|
)
|
|
|
|
result = await handle_alertmanager_resolved(
|
|
fingerprint="a" * 32,
|
|
alertname="DockerContainerUnhealthy",
|
|
severity="critical",
|
|
namespace="default",
|
|
target_resource="alertmanager",
|
|
source_alert_id="source-fp-1",
|
|
source_ends_at="2026-07-22T02:00:00Z",
|
|
correlation_lookup=correlation_lookup,
|
|
recovery_reservation=reservation,
|
|
incident_resolver=incident_resolver,
|
|
recovery_sender=sender,
|
|
outcome_recorder=outcome_recorder,
|
|
)
|
|
|
|
assert result.status == "duplicate_recovered_suppressed"
|
|
assert result.duplicate is True
|
|
assert result.provider_acknowledged is True
|
|
assert result.provider_send_performed is True
|
|
incident_resolver.assert_not_awaited()
|
|
sender.assert_not_awaited()
|
|
outcome_recorder.assert_not_awaited()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_unmatched_resolved_identity_records_no_false_recovery() -> None:
|
|
correlation_lookup = AsyncMock(return_value=None)
|
|
reservation = AsyncMock(
|
|
return_value={
|
|
"event_id": "0b7d5caf-4aa1-4e5a-8f51-987d66db7c20",
|
|
"admitted": False,
|
|
"status": "resolved_identity_unmatched",
|
|
"duplicate": True,
|
|
"delivery_attempt": 0,
|
|
}
|
|
)
|
|
incident_resolver = AsyncMock(
|
|
side_effect=AssertionError("unmatched identity must remain fail closed")
|
|
)
|
|
sender = AsyncMock(
|
|
side_effect=AssertionError("unmatched identity must not send recovery")
|
|
)
|
|
|
|
result = await handle_alertmanager_resolved(
|
|
fingerprint="b" * 32,
|
|
alertname="UnknownAlert",
|
|
severity="warning",
|
|
namespace="default",
|
|
target_resource="unknown-target",
|
|
source_alert_id="source-fp-2",
|
|
source_ends_at="2026-07-22T02:00:00Z",
|
|
correlation_lookup=correlation_lookup,
|
|
recovery_reservation=reservation,
|
|
incident_resolver=incident_resolver,
|
|
recovery_sender=sender,
|
|
outcome_recorder=AsyncMock(),
|
|
)
|
|
|
|
assert result.status == "resolved_identity_unmatched"
|
|
assert result.provider_acknowledged is False
|
|
incident_resolver.assert_not_awaited()
|
|
sender.assert_not_awaited()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_resolved_webhook_enters_canonical_lifecycle(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
lifecycle = AsyncMock(
|
|
return_value=AlertmanagerRecoveryResult(
|
|
status="recovered_acknowledged",
|
|
fingerprint="a" * 32,
|
|
incident_id="INC-20260722-ABC123",
|
|
lifecycle_event_id="0b7d5caf-4aa1-4e5a-8f51-987d66db7c20",
|
|
duplicate=False,
|
|
provider_acknowledged=True,
|
|
provider_send_performed=True,
|
|
)
|
|
)
|
|
monkeypatch.setattr(
|
|
webhooks_module,
|
|
"handle_alertmanager_resolved",
|
|
lifecycle,
|
|
)
|
|
alert = AlertmanagerAlert(
|
|
status="resolved",
|
|
labels={
|
|
"alertname": "DockerContainerUnhealthy",
|
|
"severity": "critical",
|
|
"namespace": "default",
|
|
"name": "alertmanager",
|
|
},
|
|
annotations={"summary": "container unhealthy"},
|
|
startsAt="2026-07-22T01:00:00Z",
|
|
endsAt="2026-07-22T02:00:00Z",
|
|
fingerprint="provider-fingerprint",
|
|
)
|
|
identity = _normalized_alertmanager_resolved_identity(alert)
|
|
|
|
response = await alertmanager_webhook(
|
|
_request(),
|
|
AlertmanagerPayload(status="resolved", alerts=[alert]),
|
|
BackgroundTasks(),
|
|
)
|
|
|
|
assert response.success is True
|
|
assert response.alert_id == "provider-fingerprint"
|
|
assert "provider_ack=true" in response.message
|
|
lifecycle.assert_awaited_once_with(
|
|
fingerprint=identity["fingerprint"],
|
|
alertname="DockerContainerUnhealthy",
|
|
severity="critical",
|
|
namespace="default",
|
|
target_resource="alertmanager",
|
|
source_alert_id="provider-fingerprint",
|
|
source_ends_at="2026-07-22T02:00:00Z",
|
|
)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_recovery_sender_requires_destination_bound_provider_receipt(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
from src.services import telegram_gateway as gateway_module
|
|
|
|
destination_binding = _telegram_destination_binding(-100123)
|
|
provider_result = {
|
|
"ok": True,
|
|
"result": {
|
|
"message_id": 991,
|
|
"chat": {"id": -100123},
|
|
},
|
|
"_awooop_delivery_status": "sent",
|
|
"_awooop_provider_send_performed": True,
|
|
"_awoooi_canonical_route_receipt": {
|
|
"schema_version": "telegram_canonical_egress_receipt_v1",
|
|
"decision": "allowed",
|
|
"provider_send_performed": True,
|
|
"sender_bot_alias": "tsenyang_bot",
|
|
"destination_alias": "awoooi_sre_war_room",
|
|
"destination_binding": destination_binding,
|
|
},
|
|
"_awoooi_delivery_context": {
|
|
"schema_version": "telegram_delivery_context_v1",
|
|
"sender_bot_alias": "tsenyang_bot",
|
|
"destination_alias": "awoooi_sre_war_room",
|
|
"destination_binding": destination_binding,
|
|
"payload_destination_binding": destination_binding,
|
|
"provider_destination_binding": destination_binding,
|
|
"provider_destination_verification_method": (
|
|
"requested_chat_id_matches_provider_chat_id"
|
|
),
|
|
"destination_binding_verified": True,
|
|
},
|
|
}
|
|
|
|
class _Gateway:
|
|
send_canonical_message = AsyncMock(return_value=provider_result)
|
|
|
|
gateway = _Gateway()
|
|
monkeypatch.setattr(gateway_module, "get_telegram_gateway", lambda: gateway)
|
|
|
|
delivery = await send_alertmanager_recovery(
|
|
incident_id="INC-20260722-ABC123",
|
|
alertname="DockerContainerUnhealthy",
|
|
severity="critical",
|
|
target_resource="alertmanager",
|
|
fingerprint="a" * 32,
|
|
reply_to_message_id=881,
|
|
)
|
|
|
|
assert delivery["acknowledged"] is True
|
|
assert delivery["provider_send_performed"] is True
|
|
assert delivery["ack"]["provider_message_id"] == "991"
|
|
assert delivery["ack"]["destination_binding"] == destination_binding
|
|
assert delivery["ack"]["provider_destination_binding"] == destination_binding
|
|
assert gateway.send_canonical_message.await_args.kwargs == {
|
|
"product_id": "awoooi",
|
|
"signal_family": "incident_lifecycle",
|
|
"severity": "P0",
|
|
"text": gateway.send_canonical_message.await_args.kwargs["text"],
|
|
"reply_to_message_id": 881,
|
|
}
|
|
|
|
|
|
def test_resolved_normalization_is_stable_for_same_alert_identity() -> None:
|
|
common = {
|
|
"labels": {
|
|
"alertname": "DockerContainerUnhealthy",
|
|
"severity": "critical",
|
|
"namespace": "default",
|
|
"name": "alertmanager",
|
|
},
|
|
"annotations": {"summary": "container unhealthy"},
|
|
"startsAt": "2026-07-22T01:00:00Z",
|
|
"fingerprint": "provider-fingerprint",
|
|
}
|
|
firing = _normalized_alertmanager_resolved_identity(
|
|
AlertmanagerAlert(status="firing", **common)
|
|
)
|
|
resolved = _normalized_alertmanager_resolved_identity(
|
|
AlertmanagerAlert(
|
|
status="resolved",
|
|
endsAt="2026-07-22T02:00:00Z",
|
|
**common,
|
|
)
|
|
)
|
|
|
|
assert resolved["fingerprint"] == firing["fingerprint"]
|
|
assert resolved["target_resource"] == "alertmanager"
|
|
assert resolved["namespace"] == "default"
|