fix(incidents): prefer durable closure readback

This commit is contained in:
ogt
2026-07-11 19:53:21 +08:00
parent 118b8b5ab4
commit 8a0c916d82
3 changed files with 212 additions and 5 deletions

View File

@@ -306,7 +306,7 @@ async def get_incident(incident_id: str) -> IncidentResponse:
incident_service = get_incident_service()
try:
incident = await incident_service.get_from_working_memory(incident_id)
incident = await incident_service.get_for_readback(incident_id)
if incident is None:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,

View File

@@ -664,15 +664,110 @@ class IncidentService:
)
return None
async def project_terminal_status_to_working_memory(
self,
incident_id: str,
*,
project_id: str,
status: IncidentStatus,
updated_at: datetime,
resolved_at: datetime | None,
) -> Literal["updated", "current", "missing", "failed"]:
"""Project a durable terminal DB state into the Redis read model."""
if status not in (IncidentStatus.RESOLVED, IncidentStatus.CLOSED):
raise ValueError("working_memory_terminal_projection_requires_terminal_status")
incident = await self.get_from_working_memory(incident_id)
hydrated_from_episodic = incident is None
if incident is None:
incident = await self.get_from_episodic_memory(
incident_id,
project_id=project_id,
)
if incident is None:
logger.warning(
"working_memory_terminal_projection_missing_incident",
incident_id=incident_id,
project_id=project_id,
)
return "missing"
current_updated_at = incident.updated_at
if current_updated_at.tzinfo is None:
current_updated_at = current_updated_at.replace(tzinfo=UTC)
durable_updated_at = updated_at
if durable_updated_at.tzinfo is None:
durable_updated_at = durable_updated_at.replace(tzinfo=UTC)
already_current = bool(
not hydrated_from_episodic
and incident.status in (IncidentStatus.RESOLVED, IncidentStatus.CLOSED)
and current_updated_at >= durable_updated_at
and (resolved_at is None or incident.resolved_at is not None)
)
if already_current:
return "current"
if incident.status != IncidentStatus.CLOSED:
incident.status = status
incident.updated_at = updated_at
incident.resolved_at = resolved_at or incident.resolved_at
if not await self.save_to_working_memory(incident):
logger.warning(
"working_memory_terminal_projection_failed",
incident_id=incident_id,
project_id=project_id,
)
return "failed"
logger.info(
"working_memory_terminal_projection_updated",
incident_id=incident_id,
project_id=project_id,
status=incident.status.value,
hydrated_from_episodic=hydrated_from_episodic,
)
return "updated"
async def get_for_readback(self, incident_id: str) -> Incident | None:
"""Return durable incident truth, with Redis as an availability fallback."""
incident = await self.get_from_episodic_memory(incident_id)
if incident is not None:
return incident
logger.warning(
"incident_readback_falling_back_to_working_memory",
incident_id=incident_id,
)
return await self.get_from_working_memory(incident_id)
async def get_active_incidents(self) -> list[Incident]:
"""
列出所有活躍的 Incidents (從 Working Memory)
列出所有活躍的 Incidents
方案 C: 解析時正規化舊格式 Enum 值
PostgreSQL 是 durable source of truth只有 repository 暫時不可用時,
才退回 Redis working memory 維持讀取可用性。
Returns:
list[Incident]: 活躍事件列表 (investigating 或 mitigating)
"""
try:
from src.repositories.incident_repository import get_incident_repository
incidents = await get_incident_repository().get_active()
logger.info(
"get_active_incidents_from_episodic_memory",
count=len(incidents),
)
return incidents
except Exception as e:
logger.warning(
"active_incidents_episodic_read_failed",
error=str(e),
fallback="working_memory",
)
redis_client = get_redis()
incidents: list[Incident] = []
@@ -805,7 +900,12 @@ class IncidentService:
)
return False
async def get_from_episodic_memory(self, incident_id: str) -> Incident | None:
async def get_from_episodic_memory(
self,
incident_id: str,
*,
project_id: str | None = None,
) -> Incident | None:
"""
從 Episodic Memory 讀取 Incident
@@ -813,7 +913,10 @@ class IncidentService:
Incident | None: 事件資料,若不存在則返回 None
"""
try:
async with get_db_context() as db:
db_context = (
get_db_context(project_id) if project_id else get_db_context()
)
async with db_context as db:
from sqlalchemy import select
stmt = select(IncidentRecord).where(