wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 2 Packages Projects Releases Wiki Activity

feat(alertmanager): track recovered lifecycle receipts

Browse Source
This commit is contained in:
Your Name
2026-07-22 13:51:15 +08:00
parent fb4f8e3d0c
commit 3a6f8badb7
5 changed files with 1483 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

603
apps/api/tests/test_alertmanager_recovery_lifecycle.py Normal file
View File

@@ -0,0 +1,603 @@
from __future__ import annotations
from datetime import UTC, datetime, timedelta
from unittest.mock import AsyncMock
import pytest
from fastapi import BackgroundTasks
from starlette.requests import Request
from src.api.v1 import webhooks as webhooks_module
from src.api.v1.webhooks import (
AlertmanagerAlert,
AlertmanagerPayload,
_normalized_alertmanager_resolved_identity,
alertmanager_webhook,
)
from src.services.alertmanager_recovery_lifecycle import (
AlertmanagerRecoveryResult,
_merge_reservation_observation,
_outcome_matches_current_cycle,
_reservation_decision,
handle_alertmanager_resolved,
send_alertmanager_recovery,
)
from src.services.telegram_gateway import _telegram_destination_binding
def _request() -> Request:
return Request(
{
"type": "http",
"method": "POST",
"path": "/api/v1/webhooks/alertmanager",
"scheme": "http",
"server": ("testserver", 80),
"client": ("127.0.0.1", 50000),
"query_string": b"",
"headers": [],
}
)
def test_reservation_suppresses_acknowledged_and_fresh_inflight_duplicates() -> None:
now = datetime.now(UTC)
acknowledged = _reservation_decision(
{
"status": "recovered_acknowledged",
"delivery_attempts": 1,
},
incident_id="INC-20260722-ABC123",
now=now,
)
inflight = _reservation_decision(
{
"status": "recovery_pending_delivery",
"delivery_attempts": 1,
"reserved_at": (now - timedelta(seconds=30)).isoformat(),
},
incident_id="INC-20260722-ABC123",
now=now,
)
assert acknowledged == {
"admitted": False,
"status": "duplicate_recovered_suppressed",
"attempts": 1,
}
assert inflight == {
"admitted": False,
"status": "duplicate_recovery_inflight_suppressed",
"attempts": 1,
}
def test_reservation_allows_bounded_retry_but_fails_closed_without_identity() -> None:
now = datetime.now(UTC)
retry = _reservation_decision(
{
"status": "recovery_delivery_failed",
"delivery_attempts": 1,
},
incident_id="INC-20260722-ABC123",
now=now,
)
exhausted = _reservation_decision(
{
"status": "recovery_delivery_failed",
"delivery_attempts": 3,
},
incident_id="INC-20260722-ABC123",
now=now,
)
unmatched = _reservation_decision(
None,
incident_id="",
now=now,
)
assert retry == {
"admitted": True,
"status": "recovery_pending_delivery",
"attempts": 2,
}
assert exhausted["status"] == "recovery_delivery_retry_exhausted"
assert exhausted["admitted"] is False
assert unmatched["status"] == "resolved_identity_unmatched"
assert unmatched["admitted"] is False
def test_duplicate_observation_preserves_terminal_ack_and_inflight_reservation() -> (
None
):
now = datetime.now(UTC)
acknowledged = {
"status": "recovered_acknowledged",
"delivery_attempts": 1,
"reserved_at": (now - timedelta(seconds=60)).isoformat(),
"provider_send_performed": True,
"provider_acknowledged": True,
"delivery_ack": {"provider_message_id": "991"},
}
decision = _reservation_decision(
acknowledged,
incident_id="INC-20260722-ABC123",
now=now,
)
updated = _merge_reservation_observation(
acknowledged,
decision=decision,
observed={"source_ends_at": "2026-07-22T02:00:00Z"},
now=now,
is_new=False,
)
assert updated["status"] == "recovered_acknowledged"
assert updated["reserved_at"] == acknowledged["reserved_at"]
assert updated["provider_acknowledged"] is True
assert updated["delivery_ack"] == {"provider_message_id": "991"}
assert updated["last_duplicate_status"] == "duplicate_recovered_suppressed"
inflight = {
"status": "recovery_pending_delivery",
"delivery_attempts": 1,
"reserved_at": (now - timedelta(seconds=30)).isoformat(),
"provider_send_performed": False,
"provider_acknowledged": False,
}
inflight_decision = _reservation_decision(
inflight,
incident_id="INC-20260722-ABC123",
now=now,
)
inflight_updated = _merge_reservation_observation(
inflight,
decision=inflight_decision,
observed={"source_ends_at": "2026-07-22T02:00:00Z"},
now=now,
is_new=False,
)
assert inflight_updated["status"] == "recovery_pending_delivery"
assert inflight_updated["reserved_at"] == inflight["reserved_at"]
assert inflight_updated["last_duplicate_status"] == (
"duplicate_recovery_inflight_suppressed"
)
def test_provider_send_without_destination_ack_is_never_resent_automatically() -> None:
decision = _reservation_decision(
{
"status": "recovery_delivery_failed",
"delivery_attempts": 1,
"provider_send_performed": True,
"provider_acknowledged": False,
},
incident_id="INC-20260722-ABC123",
now=datetime.now(UTC),
)
assert decision == {
"admitted": False,
"status": "recovery_delivery_ack_unresolved",
"attempts": 1,
}
def test_new_incident_reopens_same_fingerprint_without_losing_prior_ack() -> None:
now = datetime.now(UTC)
previous = {
"incident_id": "INC-20260721-OLD001",
"approval_id": "approval-old",
"status": "recovered_acknowledged",
"delivery_attempts": 1,
"provider_send_performed": True,
"provider_acknowledged": True,
"delivery_ack": {"provider_message_id": "880"},
"completed_at": (now - timedelta(days=1)).isoformat(),
}
decision = _reservation_decision(
previous,
incident_id="INC-20260722-NEW001",
now=now,
)
updated = _merge_reservation_observation(
previous,
decision=decision,
observed={
"incident_id": "INC-20260722-NEW001",
"approval_id": "approval-new",
},
now=now,
is_new=False,
)
assert decision["new_incident_cycle"] is True
assert updated["incident_id"] == "INC-20260722-NEW001"
assert updated["status"] == "recovery_pending_delivery"
assert updated["delivery_attempts"] == 1
assert updated["provider_acknowledged"] is False
assert "completed_at" not in updated
assert updated["completed_cycles"][-1]["incident_id"] == ("INC-20260721-OLD001")
assert updated["completed_cycles"][-1]["provider_acknowledged"] is True
def test_outcome_state_is_monotonic_and_cycle_bound() -> None:
lifecycle = {
"incident_id": "INC-20260722-ABC123",
"status": "recovery_pending_delivery",
"delivery_attempts": 2,
}
assert not _outcome_matches_current_cycle(
lifecycle,
incident_id="INC-20260721-OLD001",
delivery_attempt=1,
acknowledged=True,
)
assert not _outcome_matches_current_cycle(
lifecycle,
incident_id="INC-20260722-ABC123",
delivery_attempt=1,
acknowledged=False,
)
assert _outcome_matches_current_cycle(
lifecycle,
incident_id="INC-20260722-ABC123",
delivery_attempt=1,
acknowledged=True,
)
assert not _outcome_matches_current_cycle(
{**lifecycle, "status": "recovered_acknowledged"},
incident_id="INC-20260722-ABC123",
delivery_attempt=2,
acknowledged=False,
)
@pytest.mark.asyncio
async def test_resolved_handler_updates_incident_and_records_destination_ack() -> None:
correlation_lookup = AsyncMock(
return_value={
"approval_id": "approval-1",
"incident_id": "INC-20260722-ABC123",
"telegram_message_id": 881,
}
)
reservation = AsyncMock(
return_value={
"event_id": "0b7d5caf-4aa1-4e5a-8f51-987d66db7c20",
"admitted": True,
"status": "recovery_pending_delivery",
"duplicate": False,
"delivery_attempt": 1,
}
)
incident_resolver = AsyncMock(return_value=True)
sender = AsyncMock(
return_value={
"acknowledged": True,
"provider_send_performed": True,
"ack": {
"provider_message_id": "991",
"destination_binding": "binding-1",
"provider_destination_binding": "binding-1",
},
}
)
outcome_recorder = AsyncMock(return_value=None)
result = await handle_alertmanager_resolved(
fingerprint="a" * 32,
alertname="DockerContainerUnhealthy",
severity="critical",
namespace="default",
target_resource="alertmanager",
source_alert_id="source-fp-1",
source_ends_at="2026-07-22T02:00:00Z",
correlation_lookup=correlation_lookup,
recovery_reservation=reservation,
incident_resolver=incident_resolver,
recovery_sender=sender,
outcome_recorder=outcome_recorder,
)
assert result.status == "recovered_acknowledged"
assert result.provider_acknowledged is True
incident_resolver.assert_awaited_once_with("INC-20260722-ABC123")
sender.assert_awaited_once()
assert sender.await_args.kwargs["reply_to_message_id"] == 881
outcome_recorder.assert_awaited_once()
assert outcome_recorder.await_args.kwargs["delivery"]["acknowledged"] is True
@pytest.mark.asyncio
async def test_delivery_retry_uses_identity_from_durable_lifecycle_receipt() -> None:
correlation_lookup = AsyncMock(return_value=None)
reservation = AsyncMock(
return_value={
"event_id": "0b7d5caf-4aa1-4e5a-8f51-987d66db7c20",
"admitted": True,
"status": "recovery_pending_delivery",
"duplicate": False,
"delivery_attempt": 2,
"incident_id": "INC-20260722-ABC123",
"telegram_message_id": 881,
}
)
incident_resolver = AsyncMock(return_value=True)
sender = AsyncMock(
return_value={
"acknowledged": True,
"provider_send_performed": True,
"ack": {"provider_message_id": "992"},
}
)
outcome_recorder = AsyncMock(return_value=None)
result = await handle_alertmanager_resolved(
fingerprint="a" * 32,
alertname="DockerContainerUnhealthy",
severity="critical",
namespace="default",
target_resource="alertmanager",
source_alert_id="source-fp-1",
source_ends_at="2026-07-22T02:00:00Z",
correlation_lookup=correlation_lookup,
recovery_reservation=reservation,
incident_resolver=incident_resolver,
recovery_sender=sender,
outcome_recorder=outcome_recorder,
)
assert result.status == "recovered_acknowledged"
assert result.incident_id == "INC-20260722-ABC123"
incident_resolver.assert_awaited_once_with("INC-20260722-ABC123")
assert sender.await_args.kwargs["reply_to_message_id"] == 881
assert outcome_recorder.await_args.kwargs["delivery_attempt"] == 2
@pytest.mark.asyncio
async def test_duplicate_resolved_event_never_resolves_or_sends_again() -> None:
correlation_lookup = AsyncMock(
return_value={
"approval_id": "approval-1",
"incident_id": "INC-20260722-ABC123",
"telegram_message_id": 881,
}
)
reservation = AsyncMock(
return_value={
"event_id": "0b7d5caf-4aa1-4e5a-8f51-987d66db7c20",
"admitted": False,
"status": "duplicate_recovered_suppressed",
"duplicate": True,
"delivery_attempt": 1,
"provider_acknowledged": True,
"provider_send_performed": True,
}
)
incident_resolver = AsyncMock(
side_effect=AssertionError("duplicate must not resolve incident again")
)
sender = AsyncMock(
side_effect=AssertionError("duplicate must not send Telegram again")
)
outcome_recorder = AsyncMock(
side_effect=AssertionError("duplicate must not rewrite acknowledged outcome")
)
result = await handle_alertmanager_resolved(
fingerprint="a" * 32,
alertname="DockerContainerUnhealthy",
severity="critical",
namespace="default",
target_resource="alertmanager",
source_alert_id="source-fp-1",
source_ends_at="2026-07-22T02:00:00Z",
correlation_lookup=correlation_lookup,
recovery_reservation=reservation,
incident_resolver=incident_resolver,
recovery_sender=sender,
outcome_recorder=outcome_recorder,
)
assert result.status == "duplicate_recovered_suppressed"
assert result.duplicate is True
assert result.provider_acknowledged is True
assert result.provider_send_performed is True
incident_resolver.assert_not_awaited()
sender.assert_not_awaited()
outcome_recorder.assert_not_awaited()
@pytest.mark.asyncio
async def test_unmatched_resolved_identity_records_no_false_recovery() -> None:
correlation_lookup = AsyncMock(return_value=None)
reservation = AsyncMock(
return_value={
"event_id": "0b7d5caf-4aa1-4e5a-8f51-987d66db7c20",
"admitted": False,
"status": "resolved_identity_unmatched",
"duplicate": True,
"delivery_attempt": 0,
}
)
incident_resolver = AsyncMock(
side_effect=AssertionError("unmatched identity must remain fail closed")
)
sender = AsyncMock(
side_effect=AssertionError("unmatched identity must not send recovery")
)
result = await handle_alertmanager_resolved(
fingerprint="b" * 32,
alertname="UnknownAlert",
severity="warning",
namespace="default",
target_resource="unknown-target",
source_alert_id="source-fp-2",
source_ends_at="2026-07-22T02:00:00Z",
correlation_lookup=correlation_lookup,
recovery_reservation=reservation,
incident_resolver=incident_resolver,
recovery_sender=sender,
outcome_recorder=AsyncMock(),
)
assert result.status == "resolved_identity_unmatched"
assert result.provider_acknowledged is False
incident_resolver.assert_not_awaited()
sender.assert_not_awaited()
@pytest.mark.asyncio
async def test_resolved_webhook_enters_canonical_lifecycle(
monkeypatch: pytest.MonkeyPatch,
) -> None:
lifecycle = AsyncMock(
return_value=AlertmanagerRecoveryResult(
status="recovered_acknowledged",
fingerprint="a" * 32,
incident_id="INC-20260722-ABC123",
lifecycle_event_id="0b7d5caf-4aa1-4e5a-8f51-987d66db7c20",
duplicate=False,
provider_acknowledged=True,
provider_send_performed=True,
)
)
monkeypatch.setattr(
webhooks_module,
"handle_alertmanager_resolved",
lifecycle,
)
alert = AlertmanagerAlert(
status="resolved",
labels={
"alertname": "DockerContainerUnhealthy",
"severity": "critical",
"namespace": "default",
"name": "alertmanager",
},
annotations={"summary": "container unhealthy"},
startsAt="2026-07-22T01:00:00Z",
endsAt="2026-07-22T02:00:00Z",
fingerprint="provider-fingerprint",
)
identity = _normalized_alertmanager_resolved_identity(alert)
response = await alertmanager_webhook(
_request(),
AlertmanagerPayload(status="resolved", alerts=[alert]),
BackgroundTasks(),
)
assert response.success is True
assert response.alert_id == "provider-fingerprint"
assert "provider_ack=true" in response.message
lifecycle.assert_awaited_once_with(
fingerprint=identity["fingerprint"],
alertname="DockerContainerUnhealthy",
severity="critical",
namespace="default",
target_resource="alertmanager",
source_alert_id="provider-fingerprint",
source_ends_at="2026-07-22T02:00:00Z",
)
@pytest.mark.asyncio
async def test_recovery_sender_requires_destination_bound_provider_receipt(
monkeypatch: pytest.MonkeyPatch,
) -> None:
from src.services import telegram_gateway as gateway_module
destination_binding = _telegram_destination_binding(-100123)
provider_result = {
"ok": True,
"result": {
"message_id": 991,
"chat": {"id": -100123},
},
"_awooop_delivery_status": "sent",
"_awooop_provider_send_performed": True,
"_awoooi_canonical_route_receipt": {
"schema_version": "telegram_canonical_egress_receipt_v1",
"decision": "allowed",
"provider_send_performed": True,
"sender_bot_alias": "tsenyang_bot",
"destination_alias": "awoooi_sre_war_room",
"destination_binding": destination_binding,
},
"_awoooi_delivery_context": {
"schema_version": "telegram_delivery_context_v1",
"sender_bot_alias": "tsenyang_bot",
"destination_alias": "awoooi_sre_war_room",
"destination_binding": destination_binding,
"payload_destination_binding": destination_binding,
"provider_destination_binding": destination_binding,
"provider_destination_verification_method": (
"requested_chat_id_matches_provider_chat_id"
),
"destination_binding_verified": True,
},
}
class _Gateway:
send_canonical_message = AsyncMock(return_value=provider_result)
gateway = _Gateway()
monkeypatch.setattr(gateway_module, "get_telegram_gateway", lambda: gateway)
delivery = await send_alertmanager_recovery(
incident_id="INC-20260722-ABC123",
alertname="DockerContainerUnhealthy",
severity="critical",
target_resource="alertmanager",
fingerprint="a" * 32,
reply_to_message_id=881,
)
assert delivery["acknowledged"] is True
assert delivery["provider_send_performed"] is True
assert delivery["ack"]["provider_message_id"] == "991"
assert delivery["ack"]["destination_binding"] == destination_binding
assert delivery["ack"]["provider_destination_binding"] == destination_binding
assert gateway.send_canonical_message.await_args.kwargs == {
"product_id": "awoooi",
"signal_family": "incident_lifecycle",
"severity": "P0",
"text": gateway.send_canonical_message.await_args.kwargs["text"],
"reply_to_message_id": 881,
}
def test_resolved_normalization_is_stable_for_same_alert_identity() -> None:
common = {
"labels": {
"alertname": "DockerContainerUnhealthy",
"severity": "critical",
"namespace": "default",
"name": "alertmanager",
},
"annotations": {"summary": "container unhealthy"},
"startsAt": "2026-07-22T01:00:00Z",
"fingerprint": "provider-fingerprint",
}
firing = _normalized_alertmanager_resolved_identity(
AlertmanagerAlert(status="firing", **common)
)
resolved = _normalized_alertmanager_resolved_identity(
AlertmanagerAlert(
status="resolved",
endsAt="2026-07-22T02:00:00Z",
**common,
)
)
assert resolved["fingerprint"] == firing["fingerprint"]
assert resolved["target_resource"] == "alertmanager"
assert resolved["namespace"] == "default"

10
apps/api/tests/test_sre_k3s_controlled_automation_work_items_api.py
View File

@@ -155,8 +155,8 @@ def test_loader_returns_fixed_architecture_provider_order_and_agent99_bridge() -
"active_or_completed_commitments": 68,
"by_status": {
"analysis_or_governance_complete": 6,
"source_implemented_runtime_pending": 28,
"in_progress": 32,
"source_implemented_runtime_pending": 29,
"in_progress": 31,
"not_started_or_no_current_evidence": 2,
"superseded": 2,
},
@@ -176,6 +176,12 @@ def test_loader_returns_fixed_architecture_provider_order_and_agent99_bridge() -
assert "fingerprint-deduplicated" in " ".join(
commitments["AIA-CONV-030"]["source_evidence"]
)
assert commitments["AIA-CONV-031"]["status"] == (
"source_implemented_runtime_pending"
)
assert "destination-bound provider acknowledgement" in " ".join(
commitments["AIA-CONV-031"]["source_evidence"]
)
assert "Host112" in commitments["AIA-CONV-049"]["title"]
assert commitments["AIA-CONV-049"]["status"] == (
"source_implemented_runtime_pending"