awoooi/apps/api/tests/test_incident_readback_source_truth.py

"""Incident readback must not regress durable closure to stale cache state."""

from datetime import UTC, datetime
from unittest.mock import AsyncMock

import pytest
from fastapi import HTTPException

from src.api.v1 import auto_repair as auto_repair_api
from src.api.v1 import incidents as incidents_api
from src.models.incident import Incident, IncidentStatus, Severity
from src.repositories import incident_repository as incident_repository_module
from src.services.incident_service import (
    IncidentDurableReadError,
    IncidentService,
)


def _incident(status: IncidentStatus, *, minute: int) -> Incident:
    timestamp = datetime(2026, 7, 11, 11, minute, tzinfo=UTC)
    return Incident(
        incident_id="INC-20260711-SOURCE-TRUTH",
        status=status,
        severity=Severity.P2,
        affected_services=["awoooi-api"],
        created_at=datetime(2026, 7, 11, 10, 0, tzinfo=UTC),
        updated_at=timestamp,
        resolved_at=(
            timestamp
            if status in (IncidentStatus.RESOLVED, IncidentStatus.CLOSED)
            else None
        ),
        closed_at=timestamp if status is IncidentStatus.CLOSED else None,
    )


@pytest.mark.asyncio
async def test_readback_prefers_durable_resolved_incident_over_stale_cache(monkeypatch):
    service = IncidentService()
    durable = _incident(IncidentStatus.RESOLVED, minute=38)
    cached = _incident(IncidentStatus.INVESTIGATING, minute=11)
    monkeypatch.setattr(
        service,
        "_get_from_episodic_memory_strict",
        AsyncMock(return_value=durable),
    )
    working_read = AsyncMock(return_value=cached)
    monkeypatch.setattr(service, "get_from_working_memory", working_read)

    result = await service.get_for_readback(
        durable.incident_id,
        project_id="awoooi",
    )

    assert result is durable
    assert result.status is IncidentStatus.RESOLVED
    working_read.assert_not_awaited()
    service._get_from_episodic_memory_strict.assert_awaited_once_with(
        durable.incident_id,
        project_id="awoooi",
    )


@pytest.mark.asyncio
async def test_readback_returns_missing_without_trusting_working_memory(
    monkeypatch,
):
    service = IncidentService()
    cached = _incident(IncidentStatus.INVESTIGATING, minute=11)
    monkeypatch.setattr(
        service,
        "_get_from_episodic_memory_strict",
        AsyncMock(return_value=None),
    )
    working_read = AsyncMock(return_value=cached)
    monkeypatch.setattr(
        service,
        "get_from_working_memory",
        working_read,
    )

    result = await service.get_for_readback(
        cached.incident_id,
        project_id="awoooi",
    )

    assert result is None
    working_read.assert_not_awaited()


@pytest.mark.asyncio
async def test_readback_db_failure_is_degraded_not_cache_not_found(monkeypatch):
    service = IncidentService()
    working_read = AsyncMock(
        return_value=_incident(IncidentStatus.INVESTIGATING, minute=11)
    )
    monkeypatch.setattr(
        service,
        "_get_from_episodic_memory_strict",
        AsyncMock(side_effect=RuntimeError("postgres unavailable")),
    )
    monkeypatch.setattr(service, "get_from_working_memory", working_read)

    with pytest.raises(
        IncidentDurableReadError,
        match="incident_durable_readback_unavailable",
    ):
        await service.get_for_readback(
            "INC-20260711-SOURCE-TRUTH",
            project_id="awoooi",
        )

    working_read.assert_not_awaited()


@pytest.mark.asyncio
async def test_terminal_projection_replaces_newer_wrong_cache_terminal(
    monkeypatch,
):
    service = IncidentService()
    durable = _incident(IncidentStatus.CLOSED, minute=38)
    cached = _incident(IncidentStatus.RESOLVED, minute=59)
    save = AsyncMock(return_value=True)
    marker = AsyncMock(return_value=True)
    monkeypatch.setattr(
        service,
        "_get_from_episodic_memory_strict",
        AsyncMock(return_value=durable),
    )
    monkeypatch.setattr(
        service,
        "get_from_working_memory",
        AsyncMock(return_value=cached),
    )
    monkeypatch.setattr(service, "save_to_working_memory", save)
    monkeypatch.setattr(service, "_mark_terminal_projection_ready", marker)

    result = await service.project_terminal_status_to_working_memory(
        durable.incident_id,
        project_id="awoooi",
        status=IncidentStatus.CLOSED,
        updated_at=durable.updated_at,
        resolved_at=durable.resolved_at,
    )

    assert result == "updated"
    save.assert_awaited_once_with(durable)
    assert save.await_args.args[0].status is IncidentStatus.CLOSED
    marker.assert_awaited_once_with(
        incident_id=durable.incident_id,
        project_id="awoooi",
        status=IncidentStatus.CLOSED,
        updated_at=durable.updated_at,
    )


@pytest.mark.asyncio
async def test_incident_detail_endpoint_uses_durable_readback(monkeypatch):
    durable = _incident(IncidentStatus.RESOLVED, minute=38)

    class _Service:
        get_for_readback = AsyncMock(return_value=durable)

    service = _Service()
    monkeypatch.setattr(incidents_api, "get_incident_service", lambda: service)

    result = await incidents_api.get_incident(
        durable.incident_id,
        project_id="awoooi",
    )

    assert result.incident_id == durable.incident_id
    assert result.status == IncidentStatus.RESOLVED.value
    service.get_for_readback.assert_awaited_once_with(
        durable.incident_id,
        project_id="awoooi",
    )


@pytest.mark.asyncio
async def test_active_incidents_come_from_durable_repository(monkeypatch):
    service = IncidentService()
    durable_active = _incident(IncidentStatus.INVESTIGATING, minute=38)

    class _Repository:
        get_active = AsyncMock(return_value=[durable_active])

    repository = _Repository()
    monkeypatch.setattr(
        incident_repository_module,
        "get_incident_repository",
        lambda: repository,
    )

    incidents = await service.get_active_incidents(project_id="awoooi")

    assert incidents == [durable_active]
    repository.get_active.assert_awaited_once_with(project_id="awoooi")


@pytest.mark.asyncio
async def test_active_incidents_fail_closed_when_postgres_is_unavailable(
    monkeypatch,
):
    service = IncidentService()

    class _Repository:
        get_active = AsyncMock(side_effect=RuntimeError("postgres unavailable"))

    monkeypatch.setattr(
        incident_repository_module,
        "get_incident_repository",
        lambda: _Repository(),
    )

    with pytest.raises(
        IncidentDurableReadError,
        match="active_incidents_durable_read_unavailable",
    ):
        await service.get_active_incidents(project_id="awoooi")


@pytest.mark.asyncio
async def test_incident_detail_reports_durable_readback_degraded(monkeypatch):
    service = type(
        "_Service",
        (),
        {
            "get_for_readback": AsyncMock(
                side_effect=IncidentDurableReadError(
                    "incident_durable_readback_unavailable"
                )
            )
        },
    )()
    monkeypatch.setattr(incidents_api, "get_incident_service", lambda: service)

    with pytest.raises(HTTPException) as exc_info:
        await incidents_api.get_incident(
            "INC-20260711-SOURCE-TRUTH",
            project_id="awoooi",
        )

    assert exc_info.value.status_code == 503
    assert exc_info.value.detail == {
        "status": "degraded",
        "reason_code": "incident_durable_readback_unavailable",
        "source_of_truth": "postgresql",
        "redis_fallback_used": False,
    }


@pytest.mark.asyncio
async def test_repair_history_binds_project_to_durable_incident_read(monkeypatch):
    durable_active = _incident(IncidentStatus.INVESTIGATING, minute=38)

    class _Service:
        get_active_incidents = AsyncMock(return_value=[durable_active])

    service = _Service()
    monkeypatch.setattr(auto_repair_api, "get_incident_service", lambda: service)

    result = await auto_repair_api.get_repair_history(
        limit=20,
        project_id="awoooi",
    )

    assert result.count == 0
    service.get_active_incidents.assert_awaited_once_with(project_id="awoooi")


@pytest.mark.asyncio
async def test_repair_history_reports_durable_read_degraded(monkeypatch):
    class _Service:
        get_active_incidents = AsyncMock(
            side_effect=IncidentDurableReadError(
                "active_incidents_durable_read_unavailable"
            )
        )

    monkeypatch.setattr(
        auto_repair_api,
        "get_incident_service",
        lambda: _Service(),
    )

    with pytest.raises(HTTPException) as exc_info:
        await auto_repair_api.get_repair_history(
            limit=20,
            project_id="awoooi",
        )

    assert exc_info.value.status_code == 503
    assert exc_info.value.detail == {
        "status": "degraded",
        "reason_code": "active_incidents_durable_read_unavailable",
        "source_of_truth": "postgresql",
        "redis_fallback_used": False,
    }