from __future__ import annotations # ruff: noqa: E402 import json import os from pathlib import Path from types import SimpleNamespace os.environ.setdefault( "DATABASE_URL", "postgresql+asyncpg://test:test@localhost/test", ) import pytest from fastapi import FastAPI from fastapi.testclient import TestClient from pydantic import ValidationError from src.api.v1 import webhooks as webhooks_api from src.core.config import settings from src.services.alert_chain_emergency_ingress import ( ALERT_CHAIN_AGENT99_ROLE, ALERT_CHAIN_CANONICAL_ASSET_ID, ALERT_CHAIN_CATALOG_ID, ALERT_CHAIN_EXECUTOR, AlertChainEmergencyRelayRequest, build_alert_chain_emergency_owner, process_alert_chain_emergency_relay, ) REPO_ROOT = Path(__file__).resolve().parents[3] def payload() -> dict: return { "schema_version": "agent99_alertmanager_emergency_relay_v1", "relay_receipt_id": f"agent99-alertchain-{'a' * 64}", "source_host": "192.168.0.110", "source_url": "http://192.168.0.110:9093/api/v2/alerts", "source_fingerprint": "0123456789abcdef", "source_started_at": "2026-07-16T13:40:20+08:00", "alertname": "AlertChainBroken_Alertmanager", "status": "firing", "severity": "critical", "target_resource": "alertmanager", "namespace": "monitoring", "agent99_role": "relay_coordination_only", "executor": "host_ansible_executor", "catalog_id": "ansible:110-alertmanager-delivery-recovery", "runtime_write_performed": False, "raw_payload_stored": False, } class FakeApprovalService: def __init__(self) -> None: self.approval = None self.created_requests = [] self.incremented = 0 async def get_approval(self, approval_id): if self.approval and str(self.approval.id) == str(approval_id): return self.approval return None async def create_approval_with_fingerprint(self, *, request, fingerprint): self.created_requests.append((request, fingerprint)) metadata = dict(request.metadata or {}) approval_id = metadata.pop("preallocated_approval_id") self.approval = SimpleNamespace( id=approval_id, metadata=metadata, incident_id=request.incident_id, hit_count=1, ) return self.approval async def increment_hit_count(self, _approval_id): self.incremented += 1 self.approval.hit_count += 1 return self.approval async def update_incident_id(self, _approval_id, incident_id): self.approval.incident_id = incident_id class FakeIncident: def __init__(self, incident_id: str) -> None: self.incident_id = incident_id self.persisted_to_pg = True def model_dump(self, *, mode: str): assert mode == "json" return { "incident_id": self.incident_id, "status": "investigating", "severity": "P2", "affected_services": ["alertmanager"], "signals": [ { "alert_name": "AlertChainBroken_Alertmanager", "labels": { "alertname": "AlertChainBroken_Alertmanager", "component": "alertmanager", }, "annotations": {}, } ], } class FakeIncidentService: def __init__(self) -> None: self.incident = None async def get_for_readback(self, incident_id, *, project_id): assert project_id == "awoooi" if self.incident and self.incident.incident_id == incident_id: return self.incident return None def app_client() -> TestClient: app = FastAPI() app.include_router(webhooks_api.router, prefix="/api/v1") return TestClient(app) def test_relay_model_rejects_raw_content_and_executor_drift() -> None: with pytest.raises(ValidationError): AlertChainEmergencyRelayRequest.model_validate( {**payload(), "raw_log": "must not enter machine ingress"} ) with pytest.raises(ValidationError): AlertChainEmergencyRelayRequest.model_validate( {**payload(), "executor": "Agent99"} ) with pytest.raises(ValidationError): AlertChainEmergencyRelayRequest.model_validate( {**payload(), "source_started_at": "2026-07-16T13:40:20"} ) with pytest.raises(ValidationError): AlertChainEmergencyRelayRequest.model_validate( {**payload(), "source_host": "192.168.0.111"} ) def test_agent99_poll_is_read_only_first_hop_and_reduced_authenticated_outbound() -> None: source = ( REPO_ROOT / "agent99-alertmanager-alertchain-poll.ps1" ).read_text( encoding="utf-8" ) first_hop = source.split("$alerts = @(Invoke-RestMethod", 1)[1].split( "} catch", 1 )[0] assert "http://192.168.0.110:9093/api/v2/alerts" in source assert '$sourceUri.Host -ne $CanonicalSourceHost' in source assert '$sourceUri.Port -ne 9093' in source assert '$sourceUri.AbsolutePath -ne "/api/v2/alerts"' in source assert "-Method Get" in first_hop assert "-Headers" not in first_hop assert "token" not in first_hop.lower() assert '$state -ne "active"' in source assert "AlertChainBroken_Alertmanager" in source assert 'Headers @{ "X-Agent99-Relay-Token" = $token }' in source assert "agent99_alertmanager_emergency_relay_v1" in source assert "source_host = $CanonicalSourceHost" in source assert "source_url = $CanonicalSourceUrl" in source assert 'agent99_role = "relay_coordination_only"' in source assert 'executor = "host_ansible_executor"' in source assert 'catalog_id = $CanonicalCatalogId' in source assert "candidate_queued" in source assert "runtime_write_performed = $false" in source assert "raw_payload_stored = $false" in source assert "dedupe-receipts" in source assert "rawPayloadStored = $false" in source assert "Start-AgentSreAlertInbox" not in source assert "agent99-run.ps1" not in source listener = (REPO_ROOT / "agent99-sre-alert-relay.ps1").read_text( encoding="utf-8" ) assert "alertmanager-emergency" not in listener assert "AlertmanagerPrefix" not in listener def test_agent99_install_and_bootstrap_publish_poll_contract() -> None: installer = (REPO_ROOT / "agent99-install-sre-alert-relay.ps1").read_text( encoding="utf-8" ) bootstrap = (REPO_ROOT / "agent99-bootstrap.ps1").read_text( encoding="utf-8" ) for source in (installer, bootstrap): assert "http://192.168.0.110:9093/api/v2/alerts" in source assert "pollIntervalSeconds" in source assert "relay_coordination_only" in source assert "host_ansible_executor" in source assert "ansible:110-alertmanager-delivery-recovery" in source assert '"alertmanagerPrefix":' not in source assert 'Set-AgentProp $config.sreAlertRelay "alertmanagerPrefix"' not in source relay_script = (REPO_ROOT / "agent99-sre-alert-relay.ps1").read_text( encoding="utf-8" ) task_script = (REPO_ROOT / "agent99-register-tasks.ps1").read_text( encoding="utf-8" ) assert "Register-ObjectEvent" in relay_script assert "in_process_nonblocking_timer" in relay_script assert "$alertChainPollTimer.AutoReset = $false" in relay_script assert "$pollData.Timer.Start()" in relay_script assert "Unregister-Event -SourceIdentifier $alertChainPollEventId" in relay_script assert "Start-Job" not in relay_script assert "Wooo-Agent99-AlertChain-Poll" not in task_script for config_name in ( "agent99.config.example.json", "agent99.config.99.example.json", ): config = json.loads((REPO_ROOT / config_name).read_text(encoding="utf-8")) relay = config["sreAlertRelay"] assert "alertmanagerPrefix" not in relay poll = relay["alertChainPoll"] assert poll["sourceUrl"] == ( "http://192.168.0.110:9093/api/v2/alerts" ) assert poll["sourceHost"] == "192.168.0.110" assert poll["pollIntervalSeconds"] == 60 assert poll["agent99Role"] == "relay_coordination_only" assert poll["linuxExecutor"] == "host_ansible_executor" assert poll["catalogId"] == ( "ansible:110-alertmanager-delivery-recovery" ) assert poll["firstHopSecretTransmitted"] is False assert poll["rawPayloadStored"] is False def test_owner_identity_is_stable_and_binds_same_run_trace_work_item() -> None: request = AlertChainEmergencyRelayRequest.model_validate(payload()) first = build_alert_chain_emergency_owner(request) second = build_alert_chain_emergency_owner(request) assert first == second assert first["run_id"] == first["trace_id"] assert first["work_item_id"].startswith("alertchain:awoooi:") assert first["agent99_role"] == ALERT_CHAIN_AGENT99_ROLE assert first["executor"] == ALERT_CHAIN_EXECUTOR assert first["catalog_id"] == ALERT_CHAIN_CATALOG_ID assert first["canonical_asset_id"] == ALERT_CHAIN_CANONICAL_ASSET_ID @pytest.mark.asyncio async def test_secondary_ingress_persists_and_queues_exact_ansible_candidate() -> None: request = AlertChainEmergencyRelayRequest.model_validate(payload()) owner = build_alert_chain_emergency_owner(request) approvals = FakeApprovalService() incidents = FakeIncidentService() enqueue_calls = [] events = [] async def create_incident(**kwargs): assert kwargs["canonical_incident_id"] == owner["incident_id"] assert kwargs["alertname"] == "AlertChainBroken_Alertmanager" assert kwargs["namespace"] == "monitoring" incidents.incident = FakeIncident(owner["incident_id"]) return owner["incident_id"] async def enqueue(**kwargs): enqueue_calls.append(kwargs) assert kwargs["automation_run_id"] == owner["run_id"] assert kwargs["incident"]["run_id"] == owner["run_id"] assert kwargs["incident"]["trace_id"] == owner["trace_id"] assert kwargs["incident"]["work_item_id"] == owner["work_item_id"] assert kwargs["proposal_data"]["source_recurrence_verified"] is True return { "status": "controlled_check_mode_queued", "automation_run_id": owner["run_id"], "trace_id": owner["trace_id"], "work_item_id": owner["work_item_id"], "queued": True, "side_effect_performed": False, "active_blockers": [], } async def record_event(**kwargs): events.append(kwargs) result = await process_alert_chain_emergency_relay( request, approval_service=approvals, incident_service=incidents, incident_creator=create_incident, candidate_enqueuer=enqueue, event_recorder=record_event, ) assert result["status"] == "ansible_candidate_queued_verifier_pending" assert result["candidate_persisted"] is True assert result["candidate_queued"] is True assert result["identity"]["run_id"] == result["identity"]["trace_id"] assert result["identity"]["work_item_id"] == owner["work_item_id"] assert result["agent99_role"] == "relay_coordination_only" assert result["executor"] == "host_ansible_executor" assert result["runtime_write_performed"] is False assert result["runtime_closure_verified"] is False assert len(enqueue_calls) == 1 assert events[0]["stage"] == "secondary_ingress_ansible_candidate_queued" repeated = await process_alert_chain_emergency_relay( request, approval_service=approvals, incident_service=incidents, incident_creator=create_incident, candidate_enqueuer=enqueue, event_recorder=record_event, ) assert repeated["identity"] == result["identity"] assert approvals.incremented == 1 @pytest.mark.asyncio async def test_secondary_ingress_keeps_persisted_candidate_partial_when_queue_blocks() -> None: request = AlertChainEmergencyRelayRequest.model_validate(payload()) owner = build_alert_chain_emergency_owner(request) approvals = FakeApprovalService() incidents = FakeIncidentService() async def create_incident(**_kwargs): incidents.incident = FakeIncident(owner["incident_id"]) return owner["incident_id"] async def blocked_enqueue(**_kwargs): return { "status": "pre_decision_evidence_not_verified", "automation_run_id": owner["run_id"], "trace_id": owner["trace_id"], "work_item_id": owner["work_item_id"], "queued": False, "side_effect_performed": False, "active_blockers": ["pre_decision_evidence_not_verified"], } async def no_event(**_kwargs): return None result = await process_alert_chain_emergency_relay( request, approval_service=approvals, incident_service=incidents, incident_creator=create_incident, candidate_enqueuer=blocked_enqueue, event_recorder=no_event, ) assert result["candidate_persisted"] is True assert result["candidate_queued"] is False assert result["runtime_closure_verified"] is False assert result["safe_next_action"].startswith("retry_same_relay_receipt") @pytest.mark.asyncio async def test_secondary_ingress_rejects_deterministic_owner_collision() -> None: request = AlertChainEmergencyRelayRequest.model_validate(payload()) owner = build_alert_chain_emergency_owner(request) approvals = FakeApprovalService() approvals.approval = SimpleNamespace( id=owner["approval_id"], metadata={"schema_version": "untrusted_owner"}, incident_id=owner["incident_id"], hit_count=1, ) enqueue_calls = 0 async def must_not_enqueue(**_kwargs): nonlocal enqueue_calls enqueue_calls += 1 return {} result = await process_alert_chain_emergency_relay( request, approval_service=approvals, incident_service=FakeIncidentService(), candidate_enqueuer=must_not_enqueue, ) assert result["status"] == "durable_owner_identity_mismatch_no_candidate" assert result["candidate_persisted"] is False assert result["candidate_queued"] is False assert enqueue_calls == 0 def test_endpoint_requires_auth_and_fails_closed_until_queue_ack(monkeypatch) -> None: monkeypatch.setattr(settings, "AGENT99_SRE_ALERT_RELAY_TOKEN", "expected") missing = app_client().post( "/api/v1/webhooks/agent99/alertmanager-emergency", json=payload(), ) assert missing.status_code == 401 async def blocked(_payload): return { "status": "pre_decision_evidence_not_verified", "candidate_persisted": True, "candidate_queued": False, } monkeypatch.setattr( webhooks_api, "process_alert_chain_emergency_relay", blocked, ) response = app_client().post( "/api/v1/webhooks/agent99/alertmanager-emergency", headers={"X-Agent99-Relay-Token": "expected"}, json=payload(), ) assert response.status_code == 503 assert response.json()["detail"] == "pre_decision_evidence_not_verified" def test_endpoint_returns_public_same_run_receipt(monkeypatch) -> None: monkeypatch.setattr(settings, "AGENT99_SRE_ALERT_RELAY_TOKEN", "expected") request = AlertChainEmergencyRelayRequest.model_validate(payload()) owner = build_alert_chain_emergency_owner(request) async def accepted(_payload): return { "schema_version": "alert_chain_emergency_ingress_result_v1", "status": "ansible_candidate_queued_verifier_pending", "candidate_persisted": True, "candidate_queued": True, "identity": { "run_id": owner["run_id"], "trace_id": owner["trace_id"], "work_item_id": owner["work_item_id"], }, "agent99_role": "relay_coordination_only", "executor": "host_ansible_executor", "catalog_id": ALERT_CHAIN_CATALOG_ID, "runtime_write_performed": False, "runtime_closure_verified": False, } monkeypatch.setattr( webhooks_api, "process_alert_chain_emergency_relay", accepted, ) response = app_client().post( "/api/v1/webhooks/agent99/alertmanager-emergency", headers={"X-Agent99-Relay-Token": "expected"}, json=payload(), ) assert response.status_code == 202 body = response.json() assert body["identity"]["run_id"] == body["identity"]["trace_id"] assert body["agent99_role"] == "relay_coordination_only" assert body["executor"] == "host_ansible_executor" assert body["runtime_write_performed"] is False