"""Incident readback must not regress durable closure to stale cache state.""" from datetime import UTC, datetime from unittest.mock import AsyncMock import pytest from fastapi import HTTPException from src.api.v1 import incidents as incidents_api from src.models.incident import Incident, IncidentStatus, Severity from src.repositories import incident_repository as incident_repository_module from src.services.incident_service import ( IncidentDurableReadError, IncidentService, ) def _incident(status: IncidentStatus, *, minute: int) -> Incident: timestamp = datetime(2026, 7, 11, 11, minute, tzinfo=UTC) return Incident( incident_id="INC-20260711-SOURCE-TRUTH", status=status, severity=Severity.P2, affected_services=["awoooi-api"], created_at=datetime(2026, 7, 11, 10, 0, tzinfo=UTC), updated_at=timestamp, resolved_at=( timestamp if status in (IncidentStatus.RESOLVED, IncidentStatus.CLOSED) else None ), closed_at=timestamp if status is IncidentStatus.CLOSED else None, ) @pytest.mark.asyncio async def test_readback_prefers_durable_resolved_incident_over_stale_cache(monkeypatch): service = IncidentService() durable = _incident(IncidentStatus.RESOLVED, minute=38) cached = _incident(IncidentStatus.INVESTIGATING, minute=11) monkeypatch.setattr( service, "_get_from_episodic_memory_strict", AsyncMock(return_value=durable), ) working_read = AsyncMock(return_value=cached) monkeypatch.setattr(service, "get_from_working_memory", working_read) result = await service.get_for_readback( durable.incident_id, project_id="awoooi", ) assert result is durable assert result.status is IncidentStatus.RESOLVED working_read.assert_not_awaited() service._get_from_episodic_memory_strict.assert_awaited_once_with( durable.incident_id, project_id="awoooi", ) @pytest.mark.asyncio async def test_readback_returns_missing_without_trusting_working_memory( monkeypatch, ): service = IncidentService() cached = _incident(IncidentStatus.INVESTIGATING, minute=11) monkeypatch.setattr( service, "_get_from_episodic_memory_strict", AsyncMock(return_value=None), ) working_read = AsyncMock(return_value=cached) monkeypatch.setattr( service, "get_from_working_memory", working_read, ) result = await service.get_for_readback( cached.incident_id, project_id="awoooi", ) assert result is None working_read.assert_not_awaited() @pytest.mark.asyncio async def test_readback_db_failure_is_degraded_not_cache_not_found(monkeypatch): service = IncidentService() working_read = AsyncMock( return_value=_incident(IncidentStatus.INVESTIGATING, minute=11) ) monkeypatch.setattr( service, "_get_from_episodic_memory_strict", AsyncMock(side_effect=RuntimeError("postgres unavailable")), ) monkeypatch.setattr(service, "get_from_working_memory", working_read) with pytest.raises( IncidentDurableReadError, match="incident_durable_readback_unavailable", ): await service.get_for_readback( "INC-20260711-SOURCE-TRUTH", project_id="awoooi", ) working_read.assert_not_awaited() @pytest.mark.asyncio async def test_terminal_projection_replaces_newer_wrong_cache_terminal( monkeypatch, ): service = IncidentService() durable = _incident(IncidentStatus.CLOSED, minute=38) cached = _incident(IncidentStatus.RESOLVED, minute=59) save = AsyncMock(return_value=True) marker = AsyncMock(return_value=True) monkeypatch.setattr( service, "_get_from_episodic_memory_strict", AsyncMock(return_value=durable), ) monkeypatch.setattr( service, "get_from_working_memory", AsyncMock(return_value=cached), ) monkeypatch.setattr(service, "save_to_working_memory", save) monkeypatch.setattr(service, "_mark_terminal_projection_ready", marker) result = await service.project_terminal_status_to_working_memory( durable.incident_id, project_id="awoooi", status=IncidentStatus.CLOSED, updated_at=durable.updated_at, resolved_at=durable.resolved_at, ) assert result == "updated" save.assert_awaited_once_with(durable) assert save.await_args.args[0].status is IncidentStatus.CLOSED marker.assert_awaited_once_with( incident_id=durable.incident_id, project_id="awoooi", status=IncidentStatus.CLOSED, updated_at=durable.updated_at, ) @pytest.mark.asyncio async def test_incident_detail_endpoint_uses_durable_readback(monkeypatch): durable = _incident(IncidentStatus.RESOLVED, minute=38) class _Service: get_for_readback = AsyncMock(return_value=durable) service = _Service() monkeypatch.setattr(incidents_api, "get_incident_service", lambda: service) result = await incidents_api.get_incident( durable.incident_id, project_id="awoooi", ) assert result.incident_id == durable.incident_id assert result.status == IncidentStatus.RESOLVED.value service.get_for_readback.assert_awaited_once_with( durable.incident_id, project_id="awoooi", ) @pytest.mark.asyncio async def test_active_incidents_come_from_durable_repository(monkeypatch): service = IncidentService() durable_active = _incident(IncidentStatus.INVESTIGATING, minute=38) class _Repository: get_active = AsyncMock(return_value=[durable_active]) repository = _Repository() monkeypatch.setattr( incident_repository_module, "get_incident_repository", lambda: repository, ) incidents = await service.get_active_incidents(project_id="awoooi") assert incidents == [durable_active] repository.get_active.assert_awaited_once_with(project_id="awoooi") @pytest.mark.asyncio async def test_active_incidents_fail_closed_when_postgres_is_unavailable( monkeypatch, ): service = IncidentService() class _Repository: get_active = AsyncMock(side_effect=RuntimeError("postgres unavailable")) monkeypatch.setattr( incident_repository_module, "get_incident_repository", lambda: _Repository(), ) with pytest.raises( IncidentDurableReadError, match="active_incidents_durable_read_unavailable", ): await service.get_active_incidents(project_id="awoooi") @pytest.mark.asyncio async def test_incident_detail_reports_durable_readback_degraded(monkeypatch): service = type( "_Service", (), { "get_for_readback": AsyncMock( side_effect=IncidentDurableReadError( "incident_durable_readback_unavailable" ) ) }, )() monkeypatch.setattr(incidents_api, "get_incident_service", lambda: service) with pytest.raises(HTTPException) as exc_info: await incidents_api.get_incident( "INC-20260711-SOURCE-TRUTH", project_id="awoooi", ) assert exc_info.value.status_code == 503 assert exc_info.value.detail == { "status": "degraded", "reason_code": "incident_durable_readback_unavailable", "source_of_truth": "postgresql", "redis_fallback_used": False, }