from __future__ import annotations import json import pytest from src.services.agent99_sre_bridge import ( agent99_sre_single_flight_key, bridge_alertmanager_to_agent99, build_agent99_sre_alert, dispatch_agent99_sre_alert_with_receipt, post_agent99_sre_alert, release_agent99_sre_single_flight, resolve_agent99_alert_kind, write_agent99_sre_alert, ) def test_provider_freshness_alert_builds_agent99_contract() -> None: payload = build_agent99_sre_alert( alert_id="alert-20260709233000", alertname="provider_freshness_signal", severity="warning", namespace="awoooi-prod", target_resource="provider_freshness_signal", message=( "raw_signal_normalized controlled_runtime_candidate freshness " "last seen missing direct ref verifier readback" ), labels={ "service": "awoooi", "token": "must-not-leak", }, annotations={ "summary": "Provider freshness evidence is missing", }, fingerprint="fp-provider-freshness", alert_category="source_provider_freshness_triage", notification_type="raw_signal_normalized", source_url="https://awoooi.wooo.work/api/v1/webhooks/alertmanager", ) assert payload["source"] == "awoooi-api-alertmanager" assert payload["kind"] == "provider_freshness_signal" assert payload["suggestedMode"] == "ProviderFreshness" assert payload["controlledApply"] is False assert payload["force"] is True assert "provider-freshness" in payload["labels"] assert "no-false-green" in payload["labels"] assert "token=[redacted]" in payload["labels"] assert "must-not-leak" not in json.dumps(payload) assert "do not switch provider" in payload["instruction"] assert payload["routing"]["routeSource"] == "structured_kind" assert payload["routing"]["correlationKey"] == payload["routing"]["groupKey"] @pytest.mark.parametrize( ("alertname", "target", "message", "labels", "expected_kind", "expected_mode"), [ ( "HostBackupFailed", "backup", "snapshot stale", {}, "backup_health", "BackupCheck", ), ( "provider_freshness_signal", "provider_freshness_signal", "last_seen missing", {}, "provider_freshness_signal", "ProviderFreshness", ), ( "RebootAutoRecoveryActiveBlocker", "reboot-auto-recovery-slo", "all_required_hosts_not_in_10_minute_reboot_window cpu pressure", {"service": "awoooi"}, "host_recovery", "Recover", ), ( "PostgresqlDown", "postgresql", "database unavailable deployment awoooi", {"service": "awoooi"}, "database_health", "Status", ), ( "AlertChainBroken", "alertmanager", "monitoring pipeline down awoooi", {"service": "awoooi"}, "alert_chain_health", "Status", ), ( "HostDown", "192.168.0.120", "host_down vm_not_running", {"host": "192.168.0.120"}, "host_recovery", "Recover", ), ( "KubePodCrashLooping", "awoooi-api", "k3s pod crashloopbackoff", {"service": "awoooi"}, "awoooi_k3s", "AwoooRepair", ), ( "ColdStartGateBlocked", "cold-start-gate", "full-stack cold-start recovery gate requires remediation", {"service": "awoooi"}, "host_recovery", "Recover", ), ( "AWOOOICICDFailed", "AWOOOI deployment", "build-and-deploy deployment failed", {"service": "awoooi"}, "cicd_failure", "Status", ), ], ) def test_agent99_routing_replay_corpus( alertname: str, target: str, message: str, labels: dict[str, str], expected_kind: str, expected_mode: str, ) -> None: payload = build_agent99_sre_alert( alert_id=f"replay-{alertname}", alertname=alertname, severity="critical", namespace="awoooi-prod", target_resource=target, message=message, labels=labels, fingerprint=f"fp-{alertname}", ) assert payload["kind"] == expected_kind assert payload["suggestedMode"] == expected_mode def test_agent99_resolution_policy_uses_mode_verifier_only_for_stateful_alerts() -> None: cold_start = build_agent99_sre_alert( alert_id="cold-start", alertname="ColdStartGateBlocked", severity="critical", namespace="awoooi-prod", target_resource="cold-start-gate", message="cold-start gate blocked", ) cicd = build_agent99_sre_alert( alert_id="cicd-failure", alertname="AWOOOICICDFailed", severity="critical", namespace="awoooi-prod", target_resource="AWOOOI deployment", message="build-and-deploy failed", ) assert cold_start["resolution"]["policy"] == "mode_verifier_can_resolve" assert cicd["resolution"]["policy"] == "external_source_receipt_required" def test_alert_kind_mapping_for_route_and_performance() -> None: assert ( resolve_agent99_alert_kind( alertname="PublicRoute502", severity="critical", namespace="awoooi-prod", target_resource="awoooi-web", message="HTTP 502", ) == "public_route_502" ) assert ( resolve_agent99_alert_kind( alertname="HostHighCpuLoad", severity="warning", namespace="node", target_resource="192.168.0.110", message="load per core is high", ) == "performance_pressure" ) def test_write_agent99_sre_alert_to_configured_inbox(tmp_path) -> None: payload = build_agent99_sre_alert( alert_id="alert-write-smoke", alertname="HostHighCpuLoad", severity="warning", namespace="node", target_resource="192.168.0.110", message="CPU load per core is high", labels={"host": "192.168.0.110"}, ) written = write_agent99_sre_alert(payload, inbox_path=str(tmp_path)) assert written is not None data = json.loads(written.read_text(encoding="utf-8")) assert data["id"] == "awoooi-alertmanager-alert-write-smoke" assert data["kind"] == "performance_pressure" assert data["suggestedMode"] == "Perf" def test_post_agent99_sre_alert_to_relay(monkeypatch) -> None: payload = build_agent99_sre_alert( alert_id="relay-smoke", alertname="provider_freshness_signal", severity="warning", namespace="awoooi-prod", target_resource="provider_freshness_signal", message="freshness last_seen missing", labels={"service": "awoooi", "authorization": "must-not-leak"}, ) seen: dict[str, object] = {} class FakeResponse: def getcode(self) -> int: return 202 def read(self, _limit: int) -> bytes: return b'{"ok":true}' def __enter__(self) -> FakeResponse: return self def __exit__(self, *_args: object) -> None: return None def fake_urlopen(request, timeout): # type: ignore[no-untyped-def] seen["url"] = request.full_url seen["timeout"] = timeout seen["headers"] = dict(request.header_items()) seen["payload"] = json.loads(request.data.decode("utf-8")) return FakeResponse() monkeypatch.setattr( "src.services.agent99_sre_bridge.urllib.request.urlopen", fake_urlopen, ) receipt = post_agent99_sre_alert( payload, relay_url="http://192.168.0.99:8787/agent99/sre-alert/", relay_token="relay-token", timeout_seconds=1.5, ) assert receipt == {"status": 202, "body": '{"ok":true}'} assert seen["url"] == "http://192.168.0.99:8787/agent99/sre-alert/" assert seen["timeout"] == 1.5 assert seen["payload"]["kind"] == "provider_freshness_signal" assert seen["payload"]["suggestedMode"] == "ProviderFreshness" assert "authorization=[redacted]" in seen["payload"]["labels"] assert "must-not-leak" not in json.dumps(seen["payload"]) assert seen["headers"]["X-agent99-relay-token"] == "relay-token" def test_dispatch_receipt_keeps_relay_acceptance_without_raw_body(monkeypatch) -> None: payload = build_agent99_sre_alert( alert_id="receipt-smoke", alertname="ColdStartGateBlocked", severity="critical", namespace="awoooi-prod", target_resource="cold-start-gate", message="cold-start gate blocked", ) monkeypatch.setattr( "src.services.agent99_sre_bridge.settings.AGENT99_SRE_ALERT_RELAY_URL", "http://192.168.0.99:8787/agent99/sre-alert/", ) monkeypatch.setattr( "src.services.agent99_sre_bridge.post_agent99_sre_alert", lambda _payload: { "status": 202, "body": '{"ok":true,"inboxTriggered":true,"incomingFile":"private-path"}', }, ) receipt = dispatch_agent99_sre_alert_with_receipt(payload) assert receipt == { "schema_version": "agent99_sre_dispatch_receipt_v1", "status": "accepted_inbox_triggered", "transport": "relay", "alert_id": "awoooi-alertmanager-receipt-smoke", "kind": "host_recovery", "http_status": 202, "accepted": True, "inbox_triggered": True, "stores_raw_response": False, } assert "private-path" not in str(receipt) def test_agent99_single_flight_key_does_not_expose_fingerprint() -> None: key = agent99_sre_single_flight_key("private/source/fingerprint") assert key.startswith("agent99:sre_dispatch:single_flight:") assert "private" not in key @pytest.mark.asyncio async def test_agent99_bridge_suppresses_duplicate_single_flight(monkeypatch) -> None: async def duplicate(*_args, **_kwargs) -> bool: return False dispatched: list[dict[str, object]] = [] monkeypatch.setattr( "src.services.agent99_sre_bridge.try_acquire_agent99_sre_single_flight", duplicate, ) monkeypatch.setattr( "src.services.agent99_sre_bridge.dispatch_agent99_sre_alert_with_receipt", lambda payload: dispatched.append(payload) or { "status": "accepted_inbox_triggered", "transport": "relay", "accepted": True, "inbox_triggered": True, }, ) result = await bridge_alertmanager_to_agent99( alert_id="duplicate-alert", alertname="HostDown", severity="critical", namespace="node", target_resource="192.168.0.120", message="host_down", fingerprint="same-fingerprint", ) assert result["status"] == "suppressed" assert result["reason"] == "single_flight_duplicate" assert dispatched == [] @pytest.mark.asyncio async def test_agent99_bridge_dispatches_single_flight_winner(monkeypatch) -> None: async def winner(*_args, **_kwargs) -> bool: return True dispatched: list[dict[str, object]] = [] monkeypatch.setattr( "src.services.agent99_sre_bridge.try_acquire_agent99_sre_single_flight", winner, ) monkeypatch.setattr( "src.services.agent99_sre_bridge.dispatch_agent99_sre_alert_with_receipt", lambda payload: dispatched.append(payload) or { "status": "accepted_inbox_triggered", "transport": "relay", "accepted": True, "inbox_triggered": True, }, ) result = await bridge_alertmanager_to_agent99( alert_id="winner-alert", alertname="HostDown", severity="critical", namespace="node", target_resource="192.168.0.120", message="host_down", fingerprint="winner-fingerprint", ) assert result["status"] == "dispatched" assert result["dispatch"] == "relay" assert result["dispatchReceipt"]["accepted"] is True assert result["dispatchReceipt"]["inbox_triggered"] is True assert result["suggestedMode"] == "Recover" assert len(dispatched) == 1 @pytest.mark.asyncio async def test_agent99_bridge_releases_lock_when_receipt_is_not_accepted( monkeypatch, ) -> None: async def winner(*_args, **_kwargs) -> bool: return True released: list[tuple[str, str]] = [] async def release(fingerprint: str, alert_id: str) -> None: released.append((fingerprint, alert_id)) monkeypatch.setattr( "src.services.agent99_sre_bridge.try_acquire_agent99_sre_single_flight", winner, ) monkeypatch.setattr( "src.services.agent99_sre_bridge.release_agent99_sre_single_flight", release, ) monkeypatch.setattr( "src.services.agent99_sre_bridge.dispatch_agent99_sre_alert_with_receipt", lambda _payload: { "status": "relay_response_not_accepted", "transport": "relay", "accepted": False, "inbox_triggered": False, }, ) result = await bridge_alertmanager_to_agent99( alert_id="receipt-rejected", alertname="HostDown", severity="critical", namespace="node", target_resource="192.168.0.120", message="host_down", fingerprint="receipt-rejected-fingerprint", ) assert result["status"] == "failed" assert result["reason"] == "relay_response_not_accepted" assert result["dispatchReceipt"]["accepted"] is False assert released == [("receipt-rejected-fingerprint", "receipt-rejected")] @pytest.mark.asyncio async def test_agent99_bridge_releases_lock_when_transport_fails(monkeypatch) -> None: async def winner(*_args, **_kwargs) -> bool: return True released: list[tuple[str, str]] = [] async def release(fingerprint: str, alert_id: str) -> None: released.append((fingerprint, alert_id)) def fail_transport(_payload: dict[str, object]) -> dict[str, object]: raise RuntimeError("relay unavailable") monkeypatch.setattr( "src.services.agent99_sre_bridge.try_acquire_agent99_sre_single_flight", winner, ) monkeypatch.setattr( "src.services.agent99_sre_bridge.release_agent99_sre_single_flight", release, ) monkeypatch.setattr( "src.services.agent99_sre_bridge.dispatch_agent99_sre_alert_with_receipt", fail_transport, ) result = await bridge_alertmanager_to_agent99( alert_id="transport-failure", alertname="HostDown", severity="critical", namespace="node", target_resource="192.168.0.120", message="host_down", fingerprint="transport-fingerprint", ) assert result == {"status": "failed", "reason": "RuntimeError"} assert released == [("transport-fingerprint", "transport-failure")] @pytest.mark.asyncio async def test_agent99_single_flight_uses_redis_nx(monkeypatch) -> None: seen: dict[str, object] = {} class FakeRedis: async def set(self, key, value, *, ex, nx): # type: ignore[no-untyped-def] seen.update({"key": key, "value": value, "ex": ex, "nx": nx}) return True monkeypatch.setattr("src.core.redis_client.get_redis", lambda: FakeRedis()) from src.services.agent99_sre_bridge import try_acquire_agent99_sre_single_flight acquired = await try_acquire_agent99_sre_single_flight( "source-fingerprint", "alert-1" ) assert acquired is True assert str(seen["key"]).startswith("agent99:sre_dispatch:single_flight:") assert seen["value"] == "alert-1" assert seen["ex"] == 600 assert seen["nx"] is True @pytest.mark.asyncio async def test_agent99_single_flight_release_is_owner_checked(monkeypatch) -> None: seen: dict[str, object] = {} class FakeRedis: async def eval(self, script, key_count, key, owner): # type: ignore[no-untyped-def] seen.update( {"script": script, "key_count": key_count, "key": key, "owner": owner} ) return 1 monkeypatch.setattr("src.core.redis_client.get_redis", lambda: FakeRedis()) await release_agent99_sre_single_flight("source-fingerprint", "alert-1") assert "redis.call('get'" in str(seen["script"]) assert seen["key_count"] == 1 assert seen["owner"] == "alert-1"