148 lines
4.9 KiB
Python
148 lines
4.9 KiB
Python
import asyncio
|
|
from types import SimpleNamespace
|
|
from unittest.mock import AsyncMock
|
|
|
|
import pytest
|
|
|
|
from src.core.context import get_current_project_context
|
|
from src.jobs import incident_lifecycle_reconciler as reconciler
|
|
from src.jobs.incident_lifecycle_reconciler import (
|
|
LifecycleCandidate,
|
|
reconcile_stuck_incidents,
|
|
)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_reconcile_stuck_incidents_resolves_strong_evidence(monkeypatch):
|
|
service = SimpleNamespace(resolve_incident=AsyncMock(return_value=object()))
|
|
timeline = SimpleNamespace(
|
|
add_event=AsyncMock(side_effect=[{"id": "tl-1"}, {"id": "tl-2"}])
|
|
)
|
|
|
|
monkeypatch.setattr(
|
|
"src.jobs.incident_lifecycle_reconciler._fetch_candidates",
|
|
AsyncMock(
|
|
return_value=[
|
|
LifecycleCandidate(
|
|
incident_id="INC-EXEC-SUCCESS",
|
|
resolution_type="auto_repair",
|
|
reason="approval_execution_success",
|
|
),
|
|
LifecycleCandidate(
|
|
incident_id="INC-TIMEOUT",
|
|
resolution_type="timeout",
|
|
reason="approval_expired",
|
|
),
|
|
]
|
|
),
|
|
)
|
|
monkeypatch.setattr(
|
|
"src.services.incident_service.get_incident_service",
|
|
lambda: service,
|
|
)
|
|
monkeypatch.setattr(
|
|
"src.services.approval_db.get_timeline_service",
|
|
lambda: timeline,
|
|
)
|
|
|
|
resolved, errors = await reconcile_stuck_incidents(limit=2)
|
|
|
|
assert (resolved, errors) == (2, 0)
|
|
assert service.resolve_incident.await_args_list[0].args == (
|
|
"INC-EXEC-SUCCESS",
|
|
)
|
|
assert service.resolve_incident.await_args_list[0].kwargs == {
|
|
"resolution_type": "auto_repair",
|
|
"emit_postmortem": False,
|
|
}
|
|
assert service.resolve_incident.await_args_list[1].args == ("INC-TIMEOUT",)
|
|
assert service.resolve_incident.await_args_list[1].kwargs == {
|
|
"resolution_type": "timeout",
|
|
"emit_postmortem": False,
|
|
}
|
|
assert timeline.add_event.await_count == 2
|
|
assert len(timeline.add_event.await_args_list[0].kwargs["event_type"]) <= 20
|
|
assert timeline.add_event.await_args_list[0].kwargs == {
|
|
"event_type": "lifecycle_reconciled",
|
|
"status": "success",
|
|
"title": "Incident lifecycle reconciled from durable source truth",
|
|
"description": (
|
|
"reason=approval_execution_success; resolution_type=auto_repair; "
|
|
"postmortem=suppressed_batch_reconcile; telegram=not_sent"
|
|
),
|
|
"actor": "incident_lifecycle_reconciler",
|
|
"actor_role": "ai_agent",
|
|
"risk_level": "low",
|
|
"incident_id": "INC-EXEC-SUCCESS",
|
|
"project_id": "awoooi",
|
|
}
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_inactive_alert_uses_service_and_writes_timeline(monkeypatch):
|
|
service = SimpleNamespace(resolve_incident=AsyncMock(return_value=object()))
|
|
timeline = SimpleNamespace(add_event=AsyncMock(return_value={"id": "tl-inactive"}))
|
|
candidate = LifecycleCandidate(
|
|
incident_id="INC-INACTIVE",
|
|
resolution_type="timeout",
|
|
reason="inactive_alert_stale",
|
|
)
|
|
|
|
monkeypatch.setattr(
|
|
"src.jobs.incident_lifecycle_reconciler._fetch_candidates",
|
|
AsyncMock(return_value=[]),
|
|
)
|
|
monkeypatch.setattr(
|
|
"src.jobs.incident_lifecycle_reconciler._fetch_active_alertnames",
|
|
AsyncMock(return_value=set()),
|
|
)
|
|
monkeypatch.setattr(
|
|
"src.jobs.incident_lifecycle_reconciler._fetch_inactive_or_duplicate_alert_candidates",
|
|
AsyncMock(return_value=[candidate]),
|
|
)
|
|
monkeypatch.setattr(
|
|
"src.services.incident_service.get_incident_service",
|
|
lambda: service,
|
|
)
|
|
monkeypatch.setattr(
|
|
"src.services.approval_db.get_timeline_service",
|
|
lambda: timeline,
|
|
)
|
|
|
|
resolved, errors = await reconcile_stuck_incidents(limit=1)
|
|
|
|
assert (resolved, errors) == (1, 0)
|
|
service.resolve_incident.assert_awaited_once_with(
|
|
"INC-INACTIVE",
|
|
resolution_type="timeout",
|
|
emit_postmortem=False,
|
|
)
|
|
timeline.add_event.assert_awaited_once()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_reconciler_loop_binds_and_clears_project_context(monkeypatch):
|
|
observed_contexts: list[dict[str, str | None]] = []
|
|
|
|
async def fake_reconcile() -> tuple[int, int]:
|
|
observed_contexts.append(get_current_project_context())
|
|
return 0, 0
|
|
|
|
async def cancel_after_first_pass(_seconds: int) -> None:
|
|
raise asyncio.CancelledError
|
|
|
|
monkeypatch.setattr(reconciler, "reconcile_stuck_incidents", fake_reconcile)
|
|
monkeypatch.setattr(reconciler.asyncio, "sleep", cancel_after_first_pass)
|
|
|
|
with pytest.raises(asyncio.CancelledError):
|
|
await reconciler.run_incident_lifecycle_reconciler_loop()
|
|
|
|
assert observed_contexts == [
|
|
{
|
|
"project_id": "awoooi",
|
|
"source": "incident_lifecycle_reconciler",
|
|
"request_id": None,
|
|
}
|
|
]
|
|
assert get_current_project_context()["project_id"] is None
|