from __future__ import annotations import json import pytest from src.services.agent99_controlled_dispatch_ledger import ( build_agent99_dispatch_identity, build_agent99_dispatch_receipt_envelope, ) from src.services.agent99_sre_bridge import ( acquire_agent99_sre_single_flight, agent99_sre_single_flight_key, bridge_alertmanager_to_agent99, build_agent99_sre_alert, dispatch_agent99_sre_alert_with_receipt, post_agent99_sre_alert, release_agent99_sre_single_flight, resolve_agent99_alert_kind, write_agent99_sre_alert, ) def test_provider_freshness_alert_builds_agent99_contract() -> None: payload = build_agent99_sre_alert( alert_id="alert-20260709233000", alertname="provider_freshness_signal", severity="warning", namespace="awoooi-prod", target_resource="provider_freshness_signal", message=( "raw_signal_normalized controlled_runtime_candidate freshness " "last seen missing direct ref verifier readback" ), labels={ "service": "awoooi", "token": "must-not-leak", }, annotations={ "summary": "Provider freshness evidence is missing", }, fingerprint="fp-provider-freshness", alert_category="source_provider_freshness_triage", notification_type="raw_signal_normalized", source_url="https://awoooi.wooo.work/api/v1/webhooks/alertmanager", ) assert payload["source"] == "awoooi-api-alertmanager" assert payload["kind"] == "provider_freshness_signal" assert payload["suggestedMode"] == "ProviderFreshness" assert payload["controlledApply"] is False assert payload["force"] is True assert "provider-freshness" in payload["labels"] assert "no-false-green" in payload["labels"] assert "token=[redacted]" in payload["labels"] assert "must-not-leak" not in json.dumps(payload) assert "do not switch provider" in payload["instruction"] assert payload["routing"]["routeSource"] == "structured_kind" assert payload["routing"]["correlationKey"] == payload["routing"]["groupKey"] def test_explicit_backup_identity_wins_over_generic_freshness_words() -> None: payload = build_agent99_sre_alert( alert_id="backup-escrow-missing", alertname="BackupCredentialEscrowEvidenceMissing", severity="critical", namespace="awoooi-prod", target_resource="backup_restore", message=( "backup freshness missing; escrow evidence missing; " "readback verifier required" ), labels={ "event_type": "backup_restore_escrow_signal", "lane": "backup_restore_escrow_triage", }, fingerprint="fp-backup-freshness-collision", ) assert payload["kind"] == "backup_health" assert payload["suggestedMode"] == "BackupCheck" assert payload["controlledApply"] is False assert "backup-readback" in payload["labels"] assert "no-false-green" in payload["labels"] assert "read-only" in payload["labels"] assert "do not run backup/restore" in payload["instruction"] assert "remote delete" in payload["instruction"] assert payload["resolution"]["policy"] == "mode_verifier_can_resolve" @pytest.mark.parametrize( ("alertname", "target", "message", "labels", "expected_kind", "expected_mode"), [ ( "HostBackupFailed", "backup", "snapshot stale", {}, "backup_health", "BackupCheck", ), ( "provider_freshness_signal", "provider_freshness_signal", "last_seen missing", {}, "provider_freshness_signal", "ProviderFreshness", ), ( "RebootAutoRecoveryActiveBlocker", "reboot-auto-recovery-slo", "all_required_hosts_not_in_10_minute_reboot_window cpu pressure", {"service": "awoooi"}, "host_recovery", "Recover", ), ( "PostgresqlDown", "postgresql", "database unavailable deployment awoooi", {"service": "awoooi"}, "database_health", "Status", ), ( "AlertChainBroken", "alertmanager", "monitoring pipeline down awoooi", {"service": "awoooi"}, "alert_chain_health", "Status", ), ( "HostDown", "192.168.0.120", "host_down vm_not_running", {"host": "192.168.0.120"}, "host_recovery", "Recover", ), ( "Host112GuestNotReady", "host112-guest-recovery", "Host 112 console LightDM VMware Tools and Wazuh are not ready", {"host": "112", "service": "host112-guest-recovery"}, "host112_guest_recovery", "Recover", ), ( "KubePodCrashLooping", "awoooi-api", "k3s pod crashloopbackoff", {"service": "awoooi"}, "awoooi_k3s", "AwoooRepair", ), ( "ColdStartGateBlocked", "cold-start-gate", "full-stack cold-start recovery gate requires remediation", {"service": "awoooi"}, "host_recovery", "Recover", ), ( "AWOOOICICDFailed", "AWOOOI deployment", "build-and-deploy deployment failed", {"service": "awoooi"}, "cicd_failure", "Status", ), ], ) def test_agent99_routing_replay_corpus( alertname: str, target: str, message: str, labels: dict[str, str], expected_kind: str, expected_mode: str, ) -> None: payload = build_agent99_sre_alert( alert_id=f"replay-{alertname}", alertname=alertname, severity="critical", namespace="awoooi-prod", target_resource=target, message=message, labels=labels, fingerprint=f"fp-{alertname}", ) assert payload["kind"] == expected_kind assert payload["suggestedMode"] == expected_mode def test_agent99_resolution_policy_uses_mode_verifier_only_for_stateful_alerts() -> None: cold_start = build_agent99_sre_alert( alert_id="cold-start", alertname="ColdStartGateBlocked", severity="critical", namespace="awoooi-prod", target_resource="cold-start-gate", message="cold-start gate blocked", ) cicd = build_agent99_sre_alert( alert_id="cicd-failure", alertname="AWOOOICICDFailed", severity="critical", namespace="awoooi-prod", target_resource="AWOOOI deployment", message="build-and-deploy failed", ) assert cold_start["resolution"]["policy"] == "mode_verifier_can_resolve" assert cicd["resolution"]["policy"] == "external_source_receipt_required" def test_host112_guest_recovery_precedes_wazuh_security_classification() -> None: payload = build_agent99_sre_alert( alert_id="host112-guest-not-ready", alertname="Host112GuestNotReady", severity="critical", namespace="host112", target_resource="host112-guest-recovery", message="Host 112 Console VMware Tools Wazuh services are not ready", labels={"host": "112", "service": "host112-guest-recovery"}, ) assert payload["kind"] == "host112_guest_recovery" assert payload["suggestedMode"] == "Recover" assert payload["controlledApply"] is True assert payload["resolution"]["policy"] == "mode_verifier_can_resolve" assert "do not reboot" in payload["instruction"] def test_alert_kind_mapping_for_route_and_performance() -> None: assert ( resolve_agent99_alert_kind( alertname="PublicRoute502", severity="critical", namespace="awoooi-prod", target_resource="awoooi-web", message="HTTP 502", ) == "public_route_502" ) assert ( resolve_agent99_alert_kind( alertname="HostHighCpuLoad", severity="warning", namespace="node", target_resource="192.168.0.110", message="load per core is high", ) == "performance_pressure" ) def test_write_agent99_sre_alert_to_configured_inbox(tmp_path) -> None: payload = build_agent99_sre_alert( alert_id="alert-write-smoke", alertname="HostHighCpuLoad", severity="warning", namespace="node", target_resource="192.168.0.110", message="CPU load per core is high", labels={"host": "192.168.0.110"}, ) written = write_agent99_sre_alert(payload, inbox_path=str(tmp_path)) assert written is not None data = json.loads(written.read_text(encoding="utf-8")) assert data["id"] == "awoooi-alertmanager-alert-write-smoke" assert data["kind"] == "performance_pressure" assert data["suggestedMode"] == "Perf" def test_post_agent99_sre_alert_to_relay(monkeypatch) -> None: payload = build_agent99_sre_alert( alert_id="relay-smoke", alertname="provider_freshness_signal", severity="warning", namespace="awoooi-prod", target_resource="provider_freshness_signal", message="freshness last_seen missing", labels={"service": "awoooi", "authorization": "must-not-leak"}, ) seen: dict[str, object] = {} class FakeResponse: def getcode(self) -> int: return 202 def read(self, _limit: int) -> bytes: return b'{"ok":true}' def __enter__(self) -> FakeResponse: return self def __exit__(self, *_args: object) -> None: return None def fake_urlopen(request, timeout): # type: ignore[no-untyped-def] seen["url"] = request.full_url seen["timeout"] = timeout seen["headers"] = dict(request.header_items()) seen["payload"] = json.loads(request.data.decode("utf-8")) return FakeResponse() monkeypatch.setattr( "src.services.agent99_sre_bridge.urllib.request.urlopen", fake_urlopen, ) receipt = post_agent99_sre_alert( payload, relay_url="http://192.168.0.99:8787/agent99/sre-alert/", relay_token="relay-token", timeout_seconds=1.5, ) assert receipt == {"status": 202, "body": '{"ok":true}'} assert seen["url"] == "http://192.168.0.99:8787/agent99/sre-alert/" assert seen["timeout"] == 1.5 assert seen["payload"]["kind"] == "provider_freshness_signal" assert seen["payload"]["suggestedMode"] == "ProviderFreshness" assert "authorization=[redacted]" in seen["payload"]["labels"] assert "must-not-leak" not in json.dumps(seen["payload"]) assert seen["headers"]["X-agent99-relay-token"] == "relay-token" def test_dispatch_receipt_keeps_relay_acceptance_without_raw_body(monkeypatch) -> None: payload = build_agent99_sre_alert( alert_id="receipt-smoke", alertname="ColdStartGateBlocked", severity="critical", namespace="awoooi-prod", target_resource="cold-start-gate", message="cold-start gate blocked", ) monkeypatch.setattr( "src.services.agent99_sre_bridge.settings.AGENT99_SRE_ALERT_RELAY_URL", "http://192.168.0.99:8787/agent99/sre-alert/", ) monkeypatch.setattr( "src.services.agent99_sre_bridge.post_agent99_sre_alert", lambda _payload: { "status": 202, "body": '{"ok":true,"inboxTriggered":true,"incomingFile":"private-path"}', }, ) receipt = dispatch_agent99_sre_alert_with_receipt(payload) assert receipt == { "schema_version": "agent99_sre_dispatch_receipt_v1", "status": "accepted_inbox_triggered", "transport": "relay", "alert_id": "awoooi-alertmanager-receipt-smoke", "kind": "host_recovery", "http_status": 202, "accepted": True, "inbox_triggered": True, "delivery_certainty": "delivered", "stores_raw_response": False, } assert "private-path" not in str(receipt) def test_agent99_single_flight_key_does_not_expose_fingerprint() -> None: key = agent99_sre_single_flight_key("private/source/fingerprint") assert key.startswith("agent99:sre_dispatch:single_flight:") assert "private" not in key @pytest.mark.asyncio async def test_agent99_bridge_suppresses_duplicate_single_flight(monkeypatch) -> None: async def duplicate(*_args, **_kwargs) -> dict[str, object]: return {"acquired": False, "reason": "single_flight_duplicate"} dispatched: list[dict[str, object]] = [] monkeypatch.setattr( "src.services.agent99_sre_bridge.acquire_agent99_sre_single_flight", duplicate, ) monkeypatch.setattr( "src.services.agent99_sre_bridge.dispatch_agent99_sre_alert_with_receipt", lambda payload: dispatched.append(payload) or { "status": "accepted_inbox_triggered", "transport": "relay", "accepted": True, "inbox_triggered": True, }, ) result = await bridge_alertmanager_to_agent99( alert_id="duplicate-alert", alertname="UnknownObservation", severity="critical", namespace="node", target_resource="192.168.0.120", message="observation only", fingerprint="same-fingerprint", ) assert result["status"] == "suppressed" assert result["reason"] == "single_flight_duplicate" assert dispatched == [] @pytest.mark.asyncio async def test_agent99_bridge_dispatches_single_flight_winner(monkeypatch) -> None: async def winner(*_args, **_kwargs) -> dict[str, object]: return {"acquired": True, "reason": "acquired"} dispatched: list[dict[str, object]] = [] monkeypatch.setattr( "src.services.agent99_sre_bridge.acquire_agent99_sre_single_flight", winner, ) monkeypatch.setattr( "src.services.agent99_sre_bridge.dispatch_agent99_sre_alert_with_receipt", lambda payload: dispatched.append(payload) or { "status": "accepted_inbox_triggered", "transport": "relay", "accepted": True, "inbox_triggered": True, }, ) result = await bridge_alertmanager_to_agent99( alert_id="winner-alert", alertname="UnknownObservation", severity="critical", namespace="node", target_resource="192.168.0.120", message="observation only", fingerprint="winner-fingerprint", ) assert result["status"] == "dispatched" assert result["dispatch"] == "relay" assert result["dispatchReceipt"]["accepted"] is True assert result["dispatchReceipt"]["inbox_triggered"] is True assert result["suggestedMode"] == "Status" assert len(dispatched) == 1 @pytest.mark.asyncio async def test_agent99_bridge_releases_lock_when_receipt_is_not_accepted( monkeypatch, ) -> None: async def winner(*_args, **_kwargs) -> dict[str, object]: return {"acquired": True, "reason": "acquired"} released: list[tuple[str, str]] = [] async def release(fingerprint: str, alert_id: str) -> None: released.append((fingerprint, alert_id)) monkeypatch.setattr( "src.services.agent99_sre_bridge.acquire_agent99_sre_single_flight", winner, ) monkeypatch.setattr( "src.services.agent99_sre_bridge.release_agent99_sre_single_flight", release, ) monkeypatch.setattr( "src.services.agent99_sre_bridge.dispatch_agent99_sre_alert_with_receipt", lambda _payload: { "status": "relay_response_not_accepted", "transport": "relay", "accepted": False, "inbox_triggered": False, "delivery_certainty": "not_delivered", }, ) result = await bridge_alertmanager_to_agent99( alert_id="receipt-rejected", alertname="UnknownObservation", severity="critical", namespace="node", target_resource="192.168.0.120", message="observation only", fingerprint="receipt-rejected-fingerprint", ) assert result["status"] == "failed" assert result["reason"] == "relay_response_not_accepted" assert result["dispatchReceipt"]["accepted"] is False assert released == [("receipt-rejected-fingerprint", "receipt-rejected")] @pytest.mark.asyncio async def test_agent99_bridge_releases_lock_when_transport_fails(monkeypatch) -> None: async def winner(*_args, **_kwargs) -> dict[str, object]: return {"acquired": True, "reason": "acquired"} released: list[tuple[str, str]] = [] async def release(fingerprint: str, alert_id: str) -> None: released.append((fingerprint, alert_id)) def fail_transport(_payload: dict[str, object]) -> dict[str, object]: raise RuntimeError("relay unavailable") monkeypatch.setattr( "src.services.agent99_sre_bridge.acquire_agent99_sre_single_flight", winner, ) monkeypatch.setattr( "src.services.agent99_sre_bridge.release_agent99_sre_single_flight", release, ) monkeypatch.setattr( "src.services.agent99_sre_bridge.dispatch_agent99_sre_alert_with_receipt", fail_transport, ) result = await bridge_alertmanager_to_agent99( alert_id="transport-failure", alertname="UnknownObservation", severity="critical", namespace="node", target_resource="192.168.0.120", message="observation only", fingerprint="transport-fingerprint", ) assert result["status"] == "failed" assert result["reason"] == "RuntimeError" assert result["dispatchPerformed"] is False assert result["runtimeClosureVerified"] is False assert released == [("transport-fingerprint", "transport-failure")] @pytest.mark.asyncio async def test_agent99_single_flight_uses_redis_nx(monkeypatch) -> None: seen: dict[str, object] = {} class FakeRedis: async def set(self, key, value, *, ex, nx): # type: ignore[no-untyped-def] seen.update({"key": key, "value": value, "ex": ex, "nx": nx}) return True monkeypatch.setattr("src.core.redis_client.get_redis", lambda: FakeRedis()) from src.services.agent99_sre_bridge import try_acquire_agent99_sre_single_flight acquired = await try_acquire_agent99_sre_single_flight( "source-fingerprint", "alert-1" ) assert acquired is True assert str(seen["key"]).startswith("agent99:sre_dispatch:single_flight:") assert seen["value"] == "alert-1" assert seen["ex"] == 600 assert seen["nx"] is True @pytest.mark.asyncio async def test_agent99_single_flight_release_is_owner_checked(monkeypatch) -> None: seen: dict[str, object] = {} class FakeRedis: async def eval(self, script, key_count, key, owner): # type: ignore[no-untyped-def] seen.update( {"script": script, "key_count": key_count, "key": key, "owner": owner} ) return 1 monkeypatch.setattr("src.core.redis_client.get_redis", lambda: FakeRedis()) await release_agent99_sre_single_flight("source-fingerprint", "alert-1") assert "redis.call('get'" in str(seen["script"]) assert seen["key_count"] == 1 assert seen["owner"] == "alert-1" class _FakeControlledDispatchLedger: def __init__(self) -> None: self.reserved = False self.receipt: dict[str, object] | None = None self.complete_calls = 0 self.defer_calls = 0 async def reserve(self, *, identity, payload): # type: ignore[no-untyped-def] if self.reserved: return { "status": "idempotent_completed", "claimed": False, "identity": identity.public_dict(), "receipt": self.receipt, } self.reserved = True return { "status": "reserved", "claimed": True, "identity": identity.public_dict(), "receipt": None, "claim_token": "claim-attempt-1", } async def complete( # type: ignore[no-untyped-def] self, *, identity, claim_token, dispatch_receipt, **kwargs ) -> dict[str, object]: self.complete_calls += 1 self.receipt = { **build_agent99_dispatch_receipt_envelope( identity=identity, dispatch_receipt=dispatch_receipt, ), "receipt_persisted": True, } return self.receipt async def mark_delivery_attempt_started( # type: ignore[no-untyped-def] self, *, identity, claim_token ) -> dict[str, object]: self.receipt = { "status": "dispatch_delivery_unknown_reconcile_only", "identity": identity.public_dict(), "retry_policy": {"reconcile_only": True, "safe_to_retry": False}, "receipt_persisted": True, "runtime_closure_verified": False, } return self.receipt async def defer_claim_retryable( # type: ignore[no-untyped-def] self, *, identity, claim_token, reason, retry_after_seconds ) -> dict[str, object]: self.defer_calls += 1 self.receipt = { "schema_version": "agent99_controlled_dispatch_receipt_v1", "status": "dispatch_lock_retryable", "identity": identity.public_dict(), "dispatch_accepted": False, "inbox_triggered": False, "dispatch_promoted": False, "controlled_apply_authorized": False, "retry_policy": { "safe_to_retry": True, "retry_same_generation": True, "requires_generation_advance": False, "retry_after_seconds": retry_after_seconds, "reason": reason, }, "receipt_persisted": True, "runtime_closure_verified": False, } return self.receipt def test_agent99_dispatch_identity_is_stable_across_projections() -> None: alertmanager = build_agent99_dispatch_identity( project_id="awoooi", incident_id="INC-20260711-11C751", source_fingerprint="source-fingerprint", route_id="agent99_cold_start_recovery", execution_generation="1", ) telegram = build_agent99_dispatch_identity( project_id="awoooi", incident_id="INC-20260711-11C751", source_fingerprint="source-fingerprint", route_id="agent99_cold_start_recovery", execution_generation="1", ) assert telegram.run_id == alertmanager.run_id assert telegram.trace_id == alertmanager.trace_id assert telegram.idempotency_key == alertmanager.idempotency_key assert telegram.work_item_id == alertmanager.work_item_id @pytest.mark.asyncio async def test_mutating_dispatch_is_deduplicated_by_postgres_identity( monkeypatch, ) -> None: ledger = _FakeControlledDispatchLedger() dispatched: list[dict[str, object]] = [] async def acquired(*_args, **_kwargs) -> dict[str, object]: return {"acquired": True, "reason": "acquired"} monkeypatch.setattr( "src.services.agent99_sre_bridge.get_agent99_dispatch_ledger", lambda: ledger, ) monkeypatch.setattr( "src.services.agent99_sre_bridge.acquire_agent99_sre_single_flight", acquired, ) monkeypatch.setattr( "src.services.agent99_sre_bridge.dispatch_agent99_sre_alert_with_receipt", lambda payload: dispatched.append(payload) or { "status": "accepted_inbox_triggered", "transport": "relay", "alert_id": str(payload["id"]), "kind": str(payload["kind"]), "accepted": True, "inbox_triggered": True, }, ) common = { "alertname": "ColdStartGateBlocked", "severity": "critical", "namespace": "host_recovery", "target_resource": "cold-start-gate", "message": "cold-start gate blocked", "fingerprint": "source-fingerprint", "project_id": "awoooi", "incident_id": "INC-20260711-11C751", "route_id": "agent99_cold_start_recovery", } first = await bridge_alertmanager_to_agent99(alert_id="alertmanager-1", **common) second = await bridge_alertmanager_to_agent99(alert_id="telegram-card-2", **common) assert first["status"] == "dispatched" assert second["status"] == "deduplicated" assert len(dispatched) == 1 assert dispatched[0]["force"] is False assert first["identity"] == second["identity"] assert first["correlatedReceipt"]["runtime_execution_authorized"] is False assert first["correlatedReceipt"]["runtime_closure_verified"] is False assert first["correlatedReceipt"]["verifier"]["status"] == "pending" assert first["correlatedReceipt"]["learning_writeback"]["status"] == ( "pending_verifier" ) @pytest.mark.asyncio async def test_mutating_dispatch_redis_failure_performs_zero_transport( monkeypatch, ) -> None: ledger = _FakeControlledDispatchLedger() dispatched: list[dict[str, object]] = [] async def redis_unavailable(*_args, **_kwargs) -> dict[str, object]: return {"acquired": False, "reason": "redis_unavailable_fail_closed"} monkeypatch.setattr( "src.services.agent99_sre_bridge.get_agent99_dispatch_ledger", lambda: ledger, ) monkeypatch.setattr( "src.services.agent99_sre_bridge.acquire_agent99_sre_single_flight", redis_unavailable, ) monkeypatch.setattr( "src.services.agent99_sre_bridge.dispatch_agent99_sre_alert_with_receipt", lambda payload: dispatched.append(payload), ) result = await bridge_alertmanager_to_agent99( alert_id="cold-start-redis-failure", alertname="ColdStartGateBlocked", severity="critical", namespace="host_recovery", target_resource="cold-start-gate", message="cold-start gate blocked", fingerprint="redis-failure-fingerprint", incident_id="INC-20260711-11C751", route_id="agent99_cold_start_recovery", ) assert result["status"] == "retryable" assert result["reason"] == "redis_unavailable_fail_closed" assert result["dispatchPerformed"] is False assert dispatched == [] assert ledger.complete_calls == 0 assert ledger.defer_calls == 1 assert result["correlatedReceipt"]["receipt_persisted"] is True assert result["correlatedReceipt"]["runtime_closure_verified"] is False @pytest.mark.asyncio async def test_mutating_single_flight_redis_exception_is_fail_closed( monkeypatch, ) -> None: monkeypatch.setattr( "src.core.redis_client.get_redis", lambda: (_ for _ in ()).throw(RuntimeError("redis down")), ) decision = await acquire_agent99_sre_single_flight( "same-controlled-idempotency-key", "same-run-owner", fail_closed=True, ) assert decision == { "acquired": False, "reason": "redis_unavailable_fail_closed", } @pytest.mark.asyncio async def test_backupcheck_read_only_dispatch_has_durable_same_run_receipt( monkeypatch, ) -> None: ledger = _FakeControlledDispatchLedger() dispatched: list[dict[str, object]] = [] async def acquired(*_args, **_kwargs) -> dict[str, object]: return {"acquired": True, "reason": "acquired"} monkeypatch.setattr( "src.services.agent99_sre_bridge.get_agent99_dispatch_ledger", lambda: ledger, ) monkeypatch.setattr( "src.services.agent99_sre_bridge.acquire_agent99_sre_single_flight", acquired, ) monkeypatch.setattr( "src.services.agent99_sre_bridge.dispatch_agent99_sre_alert_with_receipt", lambda payload: dispatched.append(payload) or { "schema_version": "agent99_sre_dispatch_receipt_v1", "status": "accepted_inbox_triggered", "transport": "relay", "alert_id": str(payload["id"]), "kind": str(payload["kind"]), "accepted": True, "inbox_triggered": True, "delivery_certainty": "delivered", }, ) result = await bridge_alertmanager_to_agent99( alert_id="backup-readback", alertname="BackupCredentialEscrowEvidenceMissing", severity="warning", namespace="awoooi-prod", target_resource="backup_restore", message="freshness escrow restore evidence required", labels={"event_type": "backup_restore_escrow_signal"}, fingerprint="backup-fingerprint", incident_id="INC-20260711-BACKUP", route_id="agent99:backup_health:BackupCheck", ) assert result["status"] == "dispatched" assert result["identity"]["run_id"] assert result["identity"]["trace_id"] assert result["identity"]["work_item_id"] assert result["correlatedReceipt"]["receipt_persisted"] is True assert dispatched[0]["suggestedMode"] == "BackupCheck" assert dispatched[0]["controlledApply"] is False assert result["correlatedReceipt"]["controlled_apply_authorized"] is False class _DeliveryUnknownLedger: def __init__(self) -> None: self.reserved = False self.receipt: dict[str, object] | None = None async def reserve(self, *, identity, payload): # type: ignore[no-untyped-def] if self.reserved: return { "status": "idempotent_running", "claimed": False, "receipt": self.receipt, } self.reserved = True self.receipt = { "schema_version": "agent99_controlled_dispatch_receipt_v1", "status": "dispatch_reserved", "identity": identity.public_dict(), "outbox": {"status": "pending_delivery"}, } return { "status": "reserved", "claimed": True, "claim_token": "unknown-delivery-claim", "receipt": self.receipt, } async def complete(self, **_kwargs): # type: ignore[no-untyped-def] return { **(self.receipt or {}), "status": "receipt_persistence_failed_no_runtime_closure", "receipt_persisted": False, } async def mark_delivery_attempt_started( # type: ignore[no-untyped-def] self, *, identity, claim_token ) -> dict[str, object]: self.receipt = { "schema_version": "agent99_controlled_dispatch_receipt_v1", "status": "dispatch_delivery_unknown_reconcile_only", "identity": identity.public_dict(), "retry_policy": {"safe_to_retry": False, "reconcile_only": True}, "receipt_persisted": True, "runtime_closure_verified": False, } return self.receipt async def defer_claim_retryable(self, **_kwargs): # type: ignore[no-untyped-def] raise AssertionError("lock is acquired in this test") @pytest.mark.asyncio async def test_accepted_transport_with_receipt_write_failure_is_not_replayed( monkeypatch, ) -> None: ledger = _DeliveryUnknownLedger() dispatched: list[dict[str, object]] = [] async def acquired(*_args, **_kwargs) -> dict[str, object]: return {"acquired": True, "reason": "acquired"} monkeypatch.setattr( "src.services.agent99_sre_bridge.get_agent99_dispatch_ledger", lambda: ledger, ) monkeypatch.setattr( "src.services.agent99_sre_bridge.acquire_agent99_sre_single_flight", acquired, ) monkeypatch.setattr( "src.services.agent99_sre_bridge.dispatch_agent99_sre_alert_with_receipt", lambda payload: dispatched.append(payload) or { "status": "accepted_inbox_triggered", "transport": "relay", "alert_id": str(payload["id"]), "kind": str(payload["kind"]), "accepted": True, "inbox_triggered": True, "delivery_certainty": "delivered", }, ) common = { "alertname": "ColdStartGateBlocked", "severity": "critical", "namespace": "host_recovery", "target_resource": "cold-start-gate", "message": "cold-start gate blocked", "fingerprint": "unknown-delivery-fingerprint", "incident_id": "INC-20260711-11C751", "route_id": "agent99:host_recovery:Recover", } first = await bridge_alertmanager_to_agent99(alert_id="first", **common) second = await bridge_alertmanager_to_agent99(alert_id="second", **common) assert first["reason"] == "dispatch_receipt_persistence_failed" assert second["dispatchPerformed"] is False assert len(dispatched) == 1 class _GenerationAdvanceLedger(_FakeControlledDispatchLedger): async def reserve(self, *, identity, payload): # type: ignore[no-untyped-def] if identity.execution_generation == "1": return { "status": "idempotent_failed", "claimed": False, "receipt": { "status": "dispatch_not_delivered_retry_safe", "identity": identity.public_dict(), "retry_policy": { "safe_to_retry": True, "requires_generation_advance": True, "max_generation": 3, }, }, } return await super().reserve(identity=identity, payload=payload) @pytest.mark.asyncio async def test_safe_not_delivered_failure_advances_generation_before_dispatch( monkeypatch, ) -> None: ledger = _GenerationAdvanceLedger() dispatched: list[dict[str, object]] = [] async def acquired(*_args, **_kwargs) -> dict[str, object]: return {"acquired": True, "reason": "acquired"} monkeypatch.setattr( "src.services.agent99_sre_bridge.get_agent99_dispatch_ledger", lambda: ledger, ) monkeypatch.setattr( "src.services.agent99_sre_bridge.acquire_agent99_sre_single_flight", acquired, ) monkeypatch.setattr( "src.services.agent99_sre_bridge.dispatch_agent99_sre_alert_with_receipt", lambda payload: dispatched.append(payload) or { "status": "accepted_inbox_triggered", "transport": "relay", "alert_id": str(payload["id"]), "kind": str(payload["kind"]), "accepted": True, "inbox_triggered": True, "delivery_certainty": "delivered", }, ) result = await bridge_alertmanager_to_agent99( alert_id="retry-generation", alertname="ColdStartGateBlocked", severity="critical", namespace="host_recovery", target_resource="cold-start-gate", message="cold-start gate blocked", fingerprint="retry-generation-fingerprint", incident_id="INC-20260711-11C751", route_id="agent99:host_recovery:Recover", ) assert result["status"] == "dispatched" assert result["identity"]["execution_generation"] == "2" assert len(dispatched) == 1