fix(agent): fail closed on incident read drift

apps/api/tests/test_incident_readback_source_truth.py

@@ -4,11 +4,15 @@ from datetime import UTC, datetime
 from unittest.mock import AsyncMock
 
 import pytest
+from fastapi import HTTPException
 
 from src.api.v1 import incidents as incidents_api
 from src.models.incident import Incident, IncidentStatus, Severity
 from src.repositories import incident_repository as incident_repository_module
-from src.services.incident_service import IncidentService
+from src.services.incident_service import (
+    IncidentDurableReadError,
+    IncidentService,
+)
 
 
 def _incident(status: IncidentStatus, *, minute: int) -> Incident:
@@ -20,7 +24,12 @@ def _incident(status: IncidentStatus, *, minute: int) -> Incident:
         affected_services=["awoooi-api"],
         created_at=datetime(2026, 7, 11, 10, 0, tzinfo=UTC),
         updated_at=timestamp,
-        resolved_at=timestamp if status is IncidentStatus.RESOLVED else None,
+        resolved_at=(
+            timestamp
+            if status in (IncidentStatus.RESOLVED, IncidentStatus.CLOSED)
+            else None
+        ),
+        closed_at=timestamp if status is IncidentStatus.CLOSED else None,
     )
 
 
@@ -31,7 +40,7 @@ async def test_readback_prefers_durable_resolved_incident_over_stale_cache(monke
     cached = _incident(IncidentStatus.INVESTIGATING, minute=11)
     monkeypatch.setattr(
         service,
-        "get_from_episodic_memory",
+        "_get_from_episodic_memory_strict",
         AsyncMock(return_value=durable),
     )
     working_read = AsyncMock(return_value=cached)
@@ -45,27 +54,28 @@ async def test_readback_prefers_durable_resolved_incident_over_stale_cache(monke
     assert result is durable
     assert result.status is IncidentStatus.RESOLVED
     working_read.assert_not_awaited()
-    service.get_from_episodic_memory.assert_awaited_once_with(
+    service._get_from_episodic_memory_strict.assert_awaited_once_with(
         durable.incident_id,
         project_id="awoooi",
     )
 
 
 @pytest.mark.asyncio
-async def test_readback_falls_back_to_working_memory_when_durable_record_unavailable(
+async def test_readback_returns_missing_without_trusting_working_memory(
     monkeypatch,
 ):
     service = IncidentService()
     cached = _incident(IncidentStatus.INVESTIGATING, minute=11)
     monkeypatch.setattr(
         service,
-        "get_from_episodic_memory",
+        "_get_from_episodic_memory_strict",
         AsyncMock(return_value=None),
     )
+    working_read = AsyncMock(return_value=cached)
     monkeypatch.setattr(
         service,
         "get_from_working_memory",
-        AsyncMock(return_value=cached),
+        working_read,
     )
 
     result = await service.get_for_readback(
@@ -73,7 +83,74 @@ async def test_readback_falls_back_to_working_memory_when_durable_record_unavail
         project_id="awoooi",
     )
 
-    assert result is cached
+    assert result is None
+    working_read.assert_not_awaited()
+
+
+@pytest.mark.asyncio
+async def test_readback_db_failure_is_degraded_not_cache_not_found(monkeypatch):
+    service = IncidentService()
+    working_read = AsyncMock(
+        return_value=_incident(IncidentStatus.INVESTIGATING, minute=11)
+    )
+    monkeypatch.setattr(
+        service,
+        "_get_from_episodic_memory_strict",
+        AsyncMock(side_effect=RuntimeError("postgres unavailable")),
+    )
+    monkeypatch.setattr(service, "get_from_working_memory", working_read)
+
+    with pytest.raises(
+        IncidentDurableReadError,
+        match="incident_durable_readback_unavailable",
+    ):
+        await service.get_for_readback(
+            "INC-20260711-SOURCE-TRUTH",
+            project_id="awoooi",
+        )
+
+    working_read.assert_not_awaited()
+
+
+@pytest.mark.asyncio
+async def test_terminal_projection_replaces_newer_wrong_cache_terminal(
+    monkeypatch,
+):
+    service = IncidentService()
+    durable = _incident(IncidentStatus.CLOSED, minute=38)
+    cached = _incident(IncidentStatus.RESOLVED, minute=59)
+    save = AsyncMock(return_value=True)
+    marker = AsyncMock(return_value=True)
+    monkeypatch.setattr(
+        service,
+        "_get_from_episodic_memory_strict",
+        AsyncMock(return_value=durable),
+    )
+    monkeypatch.setattr(
+        service,
+        "get_from_working_memory",
+        AsyncMock(return_value=cached),
+    )
+    monkeypatch.setattr(service, "save_to_working_memory", save)
+    monkeypatch.setattr(service, "_mark_terminal_projection_ready", marker)
+
+    result = await service.project_terminal_status_to_working_memory(
+        durable.incident_id,
+        project_id="awoooi",
+        status=IncidentStatus.CLOSED,
+        updated_at=durable.updated_at,
+        resolved_at=durable.resolved_at,
+    )
+
+    assert result == "updated"
+    save.assert_awaited_once_with(durable)
+    assert save.await_args.args[0].status is IncidentStatus.CLOSED
+    marker.assert_awaited_once_with(
+        incident_id=durable.incident_id,
+        project_id="awoooi",
+        status=IncidentStatus.CLOSED,
+        updated_at=durable.updated_at,
+    )
 
 
 @pytest.mark.asyncio
@@ -118,3 +195,55 @@ async def test_active_incidents_come_from_durable_repository(monkeypatch):
 
     assert incidents == [durable_active]
     repository.get_active.assert_awaited_once_with(project_id="awoooi")
+
+
+@pytest.mark.asyncio
+async def test_active_incidents_fail_closed_when_postgres_is_unavailable(
+    monkeypatch,
+):
+    service = IncidentService()
+
+    class _Repository:
+        get_active = AsyncMock(side_effect=RuntimeError("postgres unavailable"))
+
+    monkeypatch.setattr(
+        incident_repository_module,
+        "get_incident_repository",
+        lambda: _Repository(),
+    )
+
+    with pytest.raises(
+        IncidentDurableReadError,
+        match="active_incidents_durable_read_unavailable",
+    ):
+        await service.get_active_incidents(project_id="awoooi")
+
+
+@pytest.mark.asyncio
+async def test_incident_detail_reports_durable_readback_degraded(monkeypatch):
+    service = type(
+        "_Service",
+        (),
+        {
+            "get_for_readback": AsyncMock(
+                side_effect=IncidentDurableReadError(
+                    "incident_durable_readback_unavailable"
+                )
+            )
+        },
+    )()
+    monkeypatch.setattr(incidents_api, "get_incident_service", lambda: service)
+
+    with pytest.raises(HTTPException) as exc_info:
+        await incidents_api.get_incident(
+            "INC-20260711-SOURCE-TRUTH",
+            project_id="awoooi",
+        )
+
+    assert exc_info.value.status_code == 503
+    assert exc_info.value.detail == {
+        "status": "degraded",
+        "reason_code": "incident_durable_readback_unavailable",
+        "source_of_truth": "postgresql",
+        "redis_fallback_used": False,
+    }