fix(agent99): route and dedupe alerts before dispatch

apps/api/tests/test_agent99_alert_dispatch_order_contract.py

@@ -0,0 +1,61 @@
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[3]
+SRE_INBOX = ROOT / "agent99-sre-alert-inbox.ps1"
+TELEGRAM_INBOX = ROOT / "agent99-telegram-inbox.ps1"
+
+
+def test_alertmanager_groups_before_agent99_dispatch() -> None:
+    source = (ROOT / "apps/api/src/api/v1/webhooks.py").read_text(encoding="utf-8")
+    normalized_at = source.index('"alertmanager_normalized"')
+    grouping_at = source.index("get_alert_grouping_service().evaluate", normalized_at)
+    grouped_return_at = source.index(
+        "if grouping_result and grouping_result.is_grouped", grouping_at
+    )
+    dispatch_at = source.index("bridge_alertmanager_to_agent99", grouped_return_at)
+
+    assert grouping_at < grouped_return_at < dispatch_at
+
+
+def test_agent99_bridge_has_fingerprint_single_flight_before_transport() -> None:
+    source = (ROOT / "apps/api/src/services/agent99_sre_bridge.py").read_text(
+        encoding="utf-8"
+    )
+    bridge_at = source.index("async def bridge_alertmanager_to_agent99")
+    lock_at = source.index("try_acquire_agent99_sre_single_flight", bridge_at)
+    dispatch_at = source.index("dispatch_agent99_sre_alert, payload", lock_at)
+
+    assert lock_at < dispatch_at
+
+
+def test_agent99_runtime_prioritizes_structured_route_over_text() -> None:
+    source = SRE_INBOX.read_text(encoding="utf-8")
+    route_at = source.index("function Resolve-AgentAlertRoute")
+    suggested_at = source.index("if ($suggestedMode", route_at)
+    kind_at = source.index("if ($kind -and $kindModes.ContainsKey", suggested_at)
+    fallback_at = source.index('if ($text -match "502', kind_at)
+
+    assert suggested_at < kind_at < fallback_at
+    assert '"database_health" = "Status"' in source
+    assert '"alert_chain_health" = "Status"' in source
+    assert '"host_recovery" = "Recover"' in source
+
+
+def test_agent99_runtime_has_local_correlation_dedupe_and_lock() -> None:
+    source = SRE_INBOX.read_text(encoding="utf-8")
+
+    assert '"sre-alert-single-flight.json"' in source
+    assert "[System.IO.FileShare]::None" in source
+    assert 'reason = "duplicate_single_flight"' in source
+    assert "severity_escalation_dispatch" in source
+    assert "correlationKey = $correlationKey" in source
+
+
+def test_telegram_ingress_writes_same_structured_route_contract() -> None:
+    source = TELEGRAM_INBOX.read_text(encoding="utf-8")
+
+    assert "function Get-AgentTelegramSuggestedMode" in source
+    assert 'schemaVersion = "agent99_alert_route_v1"' in source
+    assert 'routeSource = "telegram_structured_kind"' in source
+    assert '"database_health" = "Status"' in source
+    assert '"alert_chain_health" = "Status"' in source

apps/api/tests/test_agent99_sre_bridge.py

@@ -2,9 +2,14 @@ from __future__ import annotations
 
 import json
 
+import pytest
+
 from src.services.agent99_sre_bridge import (
+    agent99_sre_single_flight_key,
+    bridge_alertmanager_to_agent99,
     build_agent99_sre_alert,
     post_agent99_sre_alert,
+    release_agent99_sre_single_flight,
     resolve_agent99_alert_kind,
     write_agent99_sre_alert,
 )
@@ -37,13 +42,99 @@ def test_provider_freshness_alert_builds_agent99_contract() -> None:
     assert payload["source"] == "awoooi-api-alertmanager"
     assert payload["kind"] == "provider_freshness_signal"
     assert payload["suggestedMode"] == "ProviderFreshness"
-    assert payload["controlledApply"] is True
+    assert payload["controlledApply"] is False
     assert payload["force"] is True
     assert "provider-freshness" in payload["labels"]
     assert "no-false-green" in payload["labels"]
     assert "token=[redacted]" in payload["labels"]
     assert "must-not-leak" not in json.dumps(payload)
     assert "do not switch provider" in payload["instruction"]
+    assert payload["routing"]["routeSource"] == "structured_kind"
+    assert payload["routing"]["correlationKey"] == payload["routing"]["groupKey"]
+
+
+@pytest.mark.parametrize(
+    ("alertname", "target", "message", "labels", "expected_kind", "expected_mode"),
+    [
+        (
+            "HostBackupFailed",
+            "backup",
+            "snapshot stale",
+            {},
+            "backup_health",
+            "BackupCheck",
+        ),
+        (
+            "provider_freshness_signal",
+            "provider_freshness_signal",
+            "last_seen missing",
+            {},
+            "provider_freshness_signal",
+            "ProviderFreshness",
+        ),
+        (
+            "RebootAutoRecoveryActiveBlocker",
+            "reboot-auto-recovery-slo",
+            "all_required_hosts_not_in_10_minute_reboot_window cpu pressure",
+            {"service": "awoooi"},
+            "host_recovery",
+            "Recover",
+        ),
+        (
+            "PostgresqlDown",
+            "postgresql",
+            "database unavailable deployment awoooi",
+            {"service": "awoooi"},
+            "database_health",
+            "Status",
+        ),
+        (
+            "AlertChainBroken",
+            "alertmanager",
+            "monitoring pipeline down awoooi",
+            {"service": "awoooi"},
+            "alert_chain_health",
+            "Status",
+        ),
+        (
+            "HostDown",
+            "192.168.0.120",
+            "host_down vm_not_running",
+            {"host": "192.168.0.120"},
+            "host_recovery",
+            "Recover",
+        ),
+        (
+            "KubePodCrashLooping",
+            "awoooi-api",
+            "k3s pod crashloopbackoff",
+            {"service": "awoooi"},
+            "awoooi_k3s",
+            "AwoooRepair",
+        ),
+    ],
+)
+def test_agent99_routing_replay_corpus(
+    alertname: str,
+    target: str,
+    message: str,
+    labels: dict[str, str],
+    expected_kind: str,
+    expected_mode: str,
+) -> None:
+    payload = build_agent99_sre_alert(
+        alert_id=f"replay-{alertname}",
+        alertname=alertname,
+        severity="critical",
+        namespace="awoooi-prod",
+        target_resource=target,
+        message=message,
+        labels=labels,
+        fingerprint=f"fp-{alertname}",
+    )
+
+    assert payload["kind"] == expected_kind
+    assert payload["suggestedMode"] == expected_mode
 
 
 def test_alert_kind_mapping_for_route_and_performance() -> None:
@@ -108,7 +199,7 @@ def test_post_agent99_sre_alert_to_relay(monkeypatch) -> None:
         def read(self, _limit: int) -> bytes:
             return b'{"ok":true}'
 
-        def __enter__(self) -> "FakeResponse":
+        def __enter__(self) -> FakeResponse:
             return self
 
         def __exit__(self, *_args: object) -> None:
@@ -141,3 +232,155 @@ def test_post_agent99_sre_alert_to_relay(monkeypatch) -> None:
     assert "authorization=[redacted]" in seen["payload"]["labels"]
     assert "must-not-leak" not in json.dumps(seen["payload"])
     assert seen["headers"]["X-agent99-relay-token"] == "relay-token"
+
+
+def test_agent99_single_flight_key_does_not_expose_fingerprint() -> None:
+    key = agent99_sre_single_flight_key("private/source/fingerprint")
+
+    assert key.startswith("agent99:sre_dispatch:single_flight:")
+    assert "private" not in key
+
+
+@pytest.mark.asyncio
+async def test_agent99_bridge_suppresses_duplicate_single_flight(monkeypatch) -> None:
+    async def duplicate(*_args, **_kwargs) -> bool:
+        return False
+
+    dispatched: list[dict[str, object]] = []
+    monkeypatch.setattr(
+        "src.services.agent99_sre_bridge.try_acquire_agent99_sre_single_flight",
+        duplicate,
+    )
+    monkeypatch.setattr(
+        "src.services.agent99_sre_bridge.dispatch_agent99_sre_alert",
+        lambda payload: dispatched.append(payload) or "relay",
+    )
+
+    result = await bridge_alertmanager_to_agent99(
+        alert_id="duplicate-alert",
+        alertname="HostDown",
+        severity="critical",
+        namespace="node",
+        target_resource="192.168.0.120",
+        message="host_down",
+        fingerprint="same-fingerprint",
+    )
+
+    assert result["status"] == "suppressed"
+    assert result["reason"] == "single_flight_duplicate"
+    assert dispatched == []
+
+
+@pytest.mark.asyncio
+async def test_agent99_bridge_dispatches_single_flight_winner(monkeypatch) -> None:
+    async def winner(*_args, **_kwargs) -> bool:
+        return True
+
+    dispatched: list[dict[str, object]] = []
+    monkeypatch.setattr(
+        "src.services.agent99_sre_bridge.try_acquire_agent99_sre_single_flight",
+        winner,
+    )
+    monkeypatch.setattr(
+        "src.services.agent99_sre_bridge.dispatch_agent99_sre_alert",
+        lambda payload: dispatched.append(payload) or "relay",
+    )
+
+    result = await bridge_alertmanager_to_agent99(
+        alert_id="winner-alert",
+        alertname="HostDown",
+        severity="critical",
+        namespace="node",
+        target_resource="192.168.0.120",
+        message="host_down",
+        fingerprint="winner-fingerprint",
+    )
+
+    assert result["status"] == "dispatched"
+    assert result["dispatch"] == "relay"
+    assert result["suggestedMode"] == "Recover"
+    assert len(dispatched) == 1
+
+
+@pytest.mark.asyncio
+async def test_agent99_bridge_releases_lock_when_transport_fails(monkeypatch) -> None:
+    async def winner(*_args, **_kwargs) -> bool:
+        return True
+
+    released: list[tuple[str, str]] = []
+
+    async def release(fingerprint: str, alert_id: str) -> None:
+        released.append((fingerprint, alert_id))
+
+    def fail_transport(_payload: dict[str, object]) -> str:
+        raise RuntimeError("relay unavailable")
+
+    monkeypatch.setattr(
+        "src.services.agent99_sre_bridge.try_acquire_agent99_sre_single_flight",
+        winner,
+    )
+    monkeypatch.setattr(
+        "src.services.agent99_sre_bridge.release_agent99_sre_single_flight",
+        release,
+    )
+    monkeypatch.setattr(
+        "src.services.agent99_sre_bridge.dispatch_agent99_sre_alert",
+        fail_transport,
+    )
+
+    result = await bridge_alertmanager_to_agent99(
+        alert_id="transport-failure",
+        alertname="HostDown",
+        severity="critical",
+        namespace="node",
+        target_resource="192.168.0.120",
+        message="host_down",
+        fingerprint="transport-fingerprint",
+    )
+
+    assert result == {"status": "failed", "reason": "RuntimeError"}
+    assert released == [("transport-fingerprint", "transport-failure")]
+
+
+@pytest.mark.asyncio
+async def test_agent99_single_flight_uses_redis_nx(monkeypatch) -> None:
+    seen: dict[str, object] = {}
+
+    class FakeRedis:
+        async def set(self, key, value, *, ex, nx):  # type: ignore[no-untyped-def]
+            seen.update({"key": key, "value": value, "ex": ex, "nx": nx})
+            return True
+
+    monkeypatch.setattr("src.core.redis_client.get_redis", lambda: FakeRedis())
+
+    from src.services.agent99_sre_bridge import try_acquire_agent99_sre_single_flight
+
+    acquired = await try_acquire_agent99_sre_single_flight(
+        "source-fingerprint", "alert-1"
+    )
+
+    assert acquired is True
+    assert str(seen["key"]).startswith("agent99:sre_dispatch:single_flight:")
+    assert seen["value"] == "alert-1"
+    assert seen["ex"] == 600
+    assert seen["nx"] is True
+
+
+@pytest.mark.asyncio
+async def test_agent99_single_flight_release_is_owner_checked(monkeypatch) -> None:
+    seen: dict[str, object] = {}
+
+    class FakeRedis:
+        async def eval(self, script, key_count, key, owner):  # type: ignore[no-untyped-def]
+            seen.update(
+                {"script": script, "key_count": key_count, "key": key, "owner": owner}
+            )
+            return 1
+
+    monkeypatch.setattr("src.core.redis_client.get_redis", lambda: FakeRedis())
+
+    await release_agent99_sre_single_flight("source-fingerprint", "alert-1")
+
+    assert "redis.call('get'" in str(seen["script"])
+    assert seen["key_count"] == 1
+    assert seen["owner"] == "alert-1"

apps/api/tests/test_telegram_gateway_error_sanitizer.py

@@ -2,9 +2,9 @@ from __future__ import annotations
 
 from src.services.telegram_gateway import (
     _agent99_alert_from_ai_automation_card,
-    format_aiops_signal_alert_card,
     _outbound_source_envelope,
     _sanitize_telegram_error,
+    format_aiops_signal_alert_card,
 )
 
 
@@ -204,7 +204,8 @@ Authorization: Bearer abcdefghijklmnopqrstuvwxyz
     )
     assert payload["kind"] == "provider_freshness_signal"
     assert payload["suggestedMode"] == "ProviderFreshness"
-    assert payload["controlledApply"] is True
+    assert payload["controlledApply"] is False
+    assert payload["routing"]["routeSource"] == "structured_kind"
     assert payload["awoooi"]["alertCategory"] == "source_provider_freshness_triage"
     assert payload["awoooi"]["notificationType"] == "ai_automation_alert_card_v1"
     assert "component=telegram_gateway" in payload["labels"]