"""Incident readback must not regress durable closure to stale cache state.""" from datetime import UTC, datetime from unittest.mock import AsyncMock import pytest from fastapi import HTTPException from src.api.v1 import auto_repair as auto_repair_api from src.api.v1 import incidents as incidents_api from src.models.incident import Incident, IncidentStatus, Severity from src.repositories import incident_repository as incident_repository_module from src.services.incident_service import ( IncidentDurableReadError, IncidentService, ) def _incident(status: IncidentStatus, *, minute: int) -> Incident: timestamp = datetime(2026, 7, 11, 11, minute, tzinfo=UTC) return Incident( incident_id="INC-20260711-SOURCE-TRUTH", status=status, severity=Severity.P2, affected_services=["awoooi-api"], created_at=datetime(2026, 7, 11, 10, 0, tzinfo=UTC), updated_at=timestamp, resolved_at=( timestamp if status in (IncidentStatus.RESOLVED, IncidentStatus.CLOSED) else None ), closed_at=timestamp if status is IncidentStatus.CLOSED else None, ) @pytest.mark.asyncio async def test_readback_prefers_durable_resolved_incident_over_stale_cache(monkeypatch): service = IncidentService() durable = _incident(IncidentStatus.RESOLVED, minute=38) cached = _incident(IncidentStatus.INVESTIGATING, minute=11) monkeypatch.setattr( service, "_get_from_episodic_memory_strict", AsyncMock(return_value=durable), ) working_read = AsyncMock(return_value=cached) monkeypatch.setattr(service, "get_from_working_memory", working_read) result = await service.get_for_readback( durable.incident_id, project_id="awoooi", ) assert result is durable assert result.status is IncidentStatus.RESOLVED working_read.assert_not_awaited() service._get_from_episodic_memory_strict.assert_awaited_once_with( durable.incident_id, project_id="awoooi", ) @pytest.mark.asyncio async def test_readback_returns_missing_without_trusting_working_memory( monkeypatch, ): service = IncidentService() cached = _incident(IncidentStatus.INVESTIGATING, minute=11) monkeypatch.setattr( service, "_get_from_episodic_memory_strict", AsyncMock(return_value=None), ) working_read = AsyncMock(return_value=cached) monkeypatch.setattr( service, "get_from_working_memory", working_read, ) result = await service.get_for_readback( cached.incident_id, project_id="awoooi", ) assert result is None working_read.assert_not_awaited() @pytest.mark.asyncio async def test_readback_db_failure_is_degraded_not_cache_not_found(monkeypatch): service = IncidentService() working_read = AsyncMock( return_value=_incident(IncidentStatus.INVESTIGATING, minute=11) ) monkeypatch.setattr( service, "_get_from_episodic_memory_strict", AsyncMock(side_effect=RuntimeError("postgres unavailable")), ) monkeypatch.setattr(service, "get_from_working_memory", working_read) with pytest.raises( IncidentDurableReadError, match="incident_durable_readback_unavailable", ): await service.get_for_readback( "INC-20260711-SOURCE-TRUTH", project_id="awoooi", ) working_read.assert_not_awaited() @pytest.mark.asyncio async def test_terminal_projection_replaces_newer_wrong_cache_terminal( monkeypatch, ): service = IncidentService() durable = _incident(IncidentStatus.CLOSED, minute=38) cached = _incident(IncidentStatus.RESOLVED, minute=59) save = AsyncMock(return_value=True) marker = AsyncMock(return_value=True) monkeypatch.setattr( service, "_get_from_episodic_memory_strict", AsyncMock(return_value=durable), ) monkeypatch.setattr( service, "get_from_working_memory", AsyncMock(return_value=cached), ) monkeypatch.setattr(service, "save_to_working_memory", save) monkeypatch.setattr(service, "_mark_terminal_projection_ready", marker) result = await service.project_terminal_status_to_working_memory( durable.incident_id, project_id="awoooi", status=IncidentStatus.CLOSED, updated_at=durable.updated_at, resolved_at=durable.resolved_at, ) assert result == "updated" save.assert_awaited_once_with(durable) assert save.await_args.args[0].status is IncidentStatus.CLOSED marker.assert_awaited_once_with( incident_id=durable.incident_id, project_id="awoooi", status=IncidentStatus.CLOSED, updated_at=durable.updated_at, ) @pytest.mark.asyncio async def test_incident_detail_endpoint_uses_durable_readback(monkeypatch): durable = _incident(IncidentStatus.RESOLVED, minute=38) class _Service: get_for_readback = AsyncMock(return_value=durable) service = _Service() monkeypatch.setattr(incidents_api, "get_incident_service", lambda: service) result = await incidents_api.get_incident( durable.incident_id, project_id="awoooi", ) assert result.incident_id == durable.incident_id assert result.status == IncidentStatus.RESOLVED.value service.get_for_readback.assert_awaited_once_with( durable.incident_id, project_id="awoooi", ) @pytest.mark.asyncio async def test_active_incidents_come_from_durable_repository(monkeypatch): service = IncidentService() durable_active = _incident(IncidentStatus.INVESTIGATING, minute=38) class _Repository: get_active = AsyncMock(return_value=[durable_active]) repository = _Repository() monkeypatch.setattr( incident_repository_module, "get_incident_repository", lambda: repository, ) incidents = await service.get_active_incidents(project_id="awoooi") assert incidents == [durable_active] repository.get_active.assert_awaited_once_with(project_id="awoooi") @pytest.mark.asyncio async def test_active_incidents_fail_closed_when_postgres_is_unavailable( monkeypatch, ): service = IncidentService() class _Repository: get_active = AsyncMock(side_effect=RuntimeError("postgres unavailable")) monkeypatch.setattr( incident_repository_module, "get_incident_repository", lambda: _Repository(), ) with pytest.raises( IncidentDurableReadError, match="active_incidents_durable_read_unavailable", ): await service.get_active_incidents(project_id="awoooi") @pytest.mark.asyncio async def test_incident_detail_reports_durable_readback_degraded(monkeypatch): service = type( "_Service", (), { "get_for_readback": AsyncMock( side_effect=IncidentDurableReadError( "incident_durable_readback_unavailable" ) ) }, )() monkeypatch.setattr(incidents_api, "get_incident_service", lambda: service) with pytest.raises(HTTPException) as exc_info: await incidents_api.get_incident( "INC-20260711-SOURCE-TRUTH", project_id="awoooi", ) assert exc_info.value.status_code == 503 assert exc_info.value.detail == { "status": "degraded", "reason_code": "incident_durable_readback_unavailable", "source_of_truth": "postgresql", "redis_fallback_used": False, } @pytest.mark.asyncio async def test_repair_history_binds_project_to_durable_incident_read(monkeypatch): durable_active = _incident(IncidentStatus.INVESTIGATING, minute=38) class _Service: get_active_incidents = AsyncMock(return_value=[durable_active]) service = _Service() monkeypatch.setattr(auto_repair_api, "get_incident_service", lambda: service) result = await auto_repair_api.get_repair_history( limit=20, project_id="awoooi", ) assert result.count == 0 service.get_active_incidents.assert_awaited_once_with(project_id="awoooi") @pytest.mark.asyncio async def test_repair_history_reports_durable_read_degraded(monkeypatch): class _Service: get_active_incidents = AsyncMock( side_effect=IncidentDurableReadError( "active_incidents_durable_read_unavailable" ) ) monkeypatch.setattr( auto_repair_api, "get_incident_service", lambda: _Service(), ) with pytest.raises(HTTPException) as exc_info: await auto_repair_api.get_repair_history( limit=20, project_id="awoooi", ) assert exc_info.value.status_code == 503 assert exc_info.value.detail == { "status": "degraded", "reason_code": "active_incidents_durable_read_unavailable", "source_of_truth": "postgresql", "redis_fallback_used": False, }