fix(agent): reconcile terminal incident read models

apps/api/src/services/awooop_ansible_check_mode_service.py

@@ -2259,6 +2259,99 @@ async def _read_verified_apply_closure_prerequisites(
     }
 
 
+async def _project_incident_terminal_to_working_memory(
+    *,
+    incident_id: str,
+    project_id: str,
+    incident_status: str,
+    updated_at: datetime,
+    resolved_at: datetime | None,
+) -> str:
+    from src.models.incident import IncidentStatus
+    from src.services.incident_service import get_incident_service
+
+    normalized_status = str(incident_status or "").strip().lower()
+    if normalized_status not in {
+        IncidentStatus.RESOLVED.value,
+        IncidentStatus.CLOSED.value,
+    }:
+        return "failed"
+    return await get_incident_service().project_terminal_status_to_working_memory(
+        incident_id,
+        project_id=project_id,
+        status=IncidentStatus(normalized_status),
+        updated_at=updated_at,
+        resolved_at=resolved_at,
+    )
+
+
+async def reconcile_verified_incident_working_memory_once(
+    *,
+    project_id: str = "awoooi",
+    window_hours: int = 24,
+    limit: int = 10,
+) -> dict[str, Any]:
+    """Repair terminal DB truth that is still active in Redis working memory."""
+
+    stats: dict[str, Any] = {
+        "scanned": 0,
+        "updated": 0,
+        "current": 0,
+        "missing": 0,
+        "failed": 0,
+        "error": None,
+    }
+    try:
+        async with get_db_context(project_id) as db:
+            result = await db.execute(
+                text("""
+                    SELECT
+                        incident.incident_id,
+                        incident.status::text AS incident_status,
+                        incident.updated_at,
+                        incident.resolved_at
+                    FROM incidents incident
+                    WHERE incident.project_id = :project_id
+                      AND upper(incident.status::text) IN ('RESOLVED', 'CLOSED')
+                      AND incident.resolved_at IS NOT NULL
+                      AND cast(incident.outcome AS jsonb) #>>
+                            '{automation_terminal,incident_resolved}' = 'true'
+                      AND incident.updated_at >=
+                            NOW() - (:window_hours * INTERVAL '1 hour')
+                    ORDER BY incident.updated_at DESC
+                    LIMIT :limit
+                """),
+                {
+                    "project_id": project_id,
+                    "window_hours": max(1, int(window_hours)),
+                    "limit": max(1, int(limit)),
+                },
+            )
+            rows = [dict(row) for row in result.mappings().all()]
+
+        stats["scanned"] = len(rows)
+        for row in rows:
+            projection_status = await _project_incident_terminal_to_working_memory(
+                incident_id=str(row.get("incident_id") or ""),
+                project_id=project_id,
+                incident_status=str(row.get("incident_status") or ""),
+                updated_at=row["updated_at"],
+                resolved_at=row.get("resolved_at"),
+            )
+            if projection_status in stats:
+                stats[projection_status] += 1
+            else:
+                stats["failed"] += 1
+    except Exception as exc:
+        stats["error"] = f"{type(exc).__name__}: {exc}"[:500]
+        logger.warning(
+            "ansible_incident_working_memory_reconcile_failed",
+            project_id=project_id,
+            error_type=type(exc).__name__,
+        )
+    return stats
+
+
 async def _read_incident_closure_readback(
     claim: AnsibleCheckModeClaim,
     *,
@@ -2633,7 +2726,7 @@ async def _commit_verified_incident_closure(
                 and row.get("resolved_at") is not None
             ):
                 raise RuntimeError("verified_incident_closure_atomic_readback_failed")
-            return {
+            closure_result = {
                 **terminal,
                 "incident_status": str(row.get("incident_status") or ""),
                 "incident_row_version": (
@@ -2644,6 +2737,26 @@ async def _commit_verified_incident_closure(
                 "closure_receipt_written": True,
                 "resolved_lifecycle_written": True,
             }
+        projection_status = await _project_incident_terminal_to_working_memory(
+            incident_id=claim.incident_id,
+            project_id=project_id,
+            incident_status=closure_result["incident_status"],
+            updated_at=row["updated_at"],
+            resolved_at=row.get("resolved_at"),
+        )
+        closure_result["working_memory_projection_status"] = projection_status
+        closure_result["working_memory_projection_ready"] = (
+            projection_status in {"updated", "current"}
+        )
+        if closure_result["working_memory_projection_ready"] is not True:
+            logger.warning(
+                "ansible_verified_incident_closure_working_memory_pending",
+                automation_run_id=automation_run_id,
+                incident_id=claim.incident_id,
+                apply_op_id=apply_op_id,
+                projection_status=projection_status,
+            )
+        return closure_result
     except Exception as exc:
         logger.warning(
             "ansible_verified_incident_closure_atomic_write_failed",
@@ -2689,6 +2802,12 @@ async def _finalize_verified_apply_closure(
                 "resolved_lifecycle",
             ],
         }
+    if terminal.get("working_memory_projection_ready") is not True:
+        return {
+            "status": "working_memory_projection_pending",
+            "closed": False,
+            "missing": ["working_memory_terminal_projection"],
+        }
 
     readback = await _read_incident_closure_readback(
         claim,