from __future__ import annotations from contextlib import asynccontextmanager from unittest.mock import AsyncMock, MagicMock import pytest class _FakeMappings: def __init__(self, rows: list[dict]): self._rows = rows def all(self) -> list[dict]: return self._rows class _FakeResult: def __init__(self, rows: list[dict]): self._rows = rows def mappings(self) -> _FakeMappings: return _FakeMappings(self._rows) def _candidate_incident() -> dict: return { "incident_id": "INC-20260627-NODE110", "project_id": "awoooi", "status": "INVESTIGATING", "severity": "warning", "alertname": "NodeExporterDown", "alert_category": "infrastructure", "notification_type": "TYPE-3", "signals": [ { "alert_name": "NodeExporterDown", "labels": { "alertname": "NodeExporterDown", "instance": "node-exporter-110", }, "annotations": {}, } ], } @pytest.mark.asyncio async def test_backfill_enqueues_catalog_matched_incident(monkeypatch: pytest.MonkeyPatch) -> None: from src.jobs import awooop_ansible_candidate_backfill_job as job fake_db = AsyncMock() fake_db.execute = AsyncMock(return_value=_FakeResult([_candidate_incident()])) @asynccontextmanager async def fake_db_context(project_id: str = "awoooi"): assert project_id == "awoooi" yield fake_db recorded: list[dict] = [] call_order: list[str] = [] async def fake_evidence_collector(**kwargs): call_order.append("evidence") assert kwargs["project_id"] == "awoooi" assert kwargs["automation_run_id"] return MagicMock( snapshot_id="evidence-1", incident_id="INC-20260627-NODE110", sensors_succeeded=1, recent_logs="sanitized logs", ) async def fake_evidence_verifier(**kwargs): call_order.append("verify") assert kwargs["snapshot"].snapshot_id == "evidence-1" return True async def fake_recorder(**kwargs): call_order.append("record") recorded.append(kwargs) return True async def fake_receipt_backfiller(**kwargs): call_order.append("receipt") assert kwargs["project_id"] == "awoooi" return {"written": 2, "error": None} async def fake_retry_replayer(**kwargs): call_order.append("retry") assert kwargs["project_id"] == "awoooi" return { "scanned": 0, "replayed": 0, "check_mode_passed": 0, "check_mode_failed": 0, "runtime_stage_receipt_written": 0, "blockers": [], "error": None, } monkeypatch.setattr(job, "get_db_context", fake_db_context) monkeypatch.setattr(job.settings, "ENABLE_AWOOOP_ANSIBLE_CANDIDATE_BACKFILL_WORKER", True) result = await job.enqueue_missing_ansible_candidates_once( project_id="awoooi", limit=5, window_hours=24, recorder=fake_recorder, receipt_backfiller=fake_receipt_backfiller, retry_replayer=fake_retry_replayer, evidence_collector=fake_evidence_collector, evidence_verifier=fake_evidence_verifier, ) assert result["queued"] == 1 assert result["no_catalog_candidate"] == 0 assert result["repair_receipts_backfilled"] == 2 assert result["failed_apply_retry_replayed"] == 0 assert result["failed_apply_retry_check_mode_passed"] == 0 assert result["failed_apply_retry_check_mode_failed"] == 0 assert result["failed_apply_retry_stage_receipt_written"] == 0 assert result["failed_apply_retry_blockers"] == [] assert result["failed_apply_retry_priority_tick"] is False assert result["pre_decision_evidence_ready"] == 1 assert result["pre_decision_evidence_blocked"] == 0 assert call_order == ["retry", "receipt", "evidence", "verify", "record"] assert recorded[0]["decision_path"] == "repair_candidate_controlled_queue" assert recorded[0]["incident"]["incident_id"] == "INC-20260627-NODE110" assert recorded[0]["automation_run_id"] @pytest.mark.asyncio async def test_backfill_runs_retry_then_continues_fresh_candidates( monkeypatch: pytest.MonkeyPatch, ) -> None: from src.jobs import awooop_ansible_candidate_backfill_job as job async def fake_retry_replayer(**kwargs): assert kwargs["project_id"] == "awoooi" return { "scanned": 1, "replayed": 1, "check_mode_passed": 1, "check_mode_failed": 0, "runtime_stage_receipt_written": 1, "blockers": [], "error": None, } fresh_scan = AsyncMock(return_value=[]) monkeypatch.setattr(job.settings, "ENABLE_AWOOOP_ANSIBLE_CANDIDATE_BACKFILL_WORKER", True) monkeypatch.setattr(job, "_fetch_missing_candidate_incidents", fresh_scan) receipt_backfiller = AsyncMock(return_value={"written": 0, "error": None}) recorder = AsyncMock(return_value=True) result = await job.enqueue_missing_ansible_candidates_once( retry_replayer=fake_retry_replayer, receipt_backfiller=receipt_backfiller, recorder=recorder, ) assert result["failed_apply_retry_scanned"] == 1 assert result["failed_apply_retry_replayed"] == 1 assert result["failed_apply_retry_check_mode_passed"] == 1 assert result["failed_apply_retry_stage_receipt_written"] == 1 assert result["failed_apply_retry_priority_tick"] is True assert result["queued"] == 0 receipt_backfiller.assert_awaited_once() fresh_scan.assert_awaited_once() recorder.assert_not_awaited() @pytest.mark.asyncio async def test_backfill_surfaces_retry_blockers_and_continues_fresh_scan( monkeypatch: pytest.MonkeyPatch, ) -> None: from src.jobs import awooop_ansible_candidate_backfill_job as job async def fake_retry_replayer(**_kwargs): return { "scanned": 0, "replayed": 0, "blockers": ["database_connection_budget_exhausted"], "error": None, } fresh_scan = AsyncMock(return_value=[]) monkeypatch.setattr(job.settings, "ENABLE_AWOOOP_ANSIBLE_CANDIDATE_BACKFILL_WORKER", True) monkeypatch.setattr(job, "_fetch_missing_candidate_incidents", fresh_scan) result = await job.enqueue_missing_ansible_candidates_once( retry_replayer=fake_retry_replayer, receipt_backfiller=AsyncMock(return_value={"written": 0, "error": None}), ) assert result["failed_apply_retry_blockers"] == [ "database_connection_budget_exhausted" ] assert result["failed_apply_retry_priority_tick"] is True assert result["queued"] == 0 fresh_scan.assert_awaited_once() @pytest.mark.asyncio async def test_backfill_retries_automatically_when_predecision_evidence_is_incomplete( monkeypatch: pytest.MonkeyPatch, ) -> None: from src.jobs import awooop_ansible_candidate_backfill_job as job fake_db = AsyncMock() fake_db.execute = AsyncMock(return_value=_FakeResult([_candidate_incident()])) @asynccontextmanager async def fake_db_context(project_id: str = "awoooi"): yield fake_db async def fake_evidence_collector(**_kwargs): return None async def fake_evidence_verifier(**kwargs): assert kwargs["snapshot"] is None return False recorder = AsyncMock(return_value=True) monkeypatch.setattr(job, "get_db_context", fake_db_context) monkeypatch.setattr(job.settings, "ENABLE_AWOOOP_ANSIBLE_CANDIDATE_BACKFILL_WORKER", True) result = await job.enqueue_missing_ansible_candidates_once( recorder=recorder, evidence_collector=fake_evidence_collector, evidence_verifier=fake_evidence_verifier, receipt_backfiller=AsyncMock(return_value={"written": 0, "error": None}), retry_replayer=AsyncMock(return_value={"error": None}), ) assert result["queued"] == 0 assert result["pre_decision_evidence_blocked"] == 1 recorder.assert_not_awaited() @pytest.mark.asyncio async def test_predecision_verifier_casts_json_health_to_jsonb( monkeypatch: pytest.MonkeyPatch, ) -> None: from src.jobs import awooop_ansible_candidate_backfill_job as job statements: list[str] = [] class ScalarResult: def scalar(self): return True class FakeDb: async def execute(self, statement, _parameters): statements.append(str(statement)) return ScalarResult() @asynccontextmanager async def fake_db_context(project_id: str = "awoooi"): assert project_id == "awoooi" yield FakeDb() monkeypatch.setattr(job, "get_db_context", fake_db_context) snapshot = MagicMock( snapshot_id="evidence-1", incident_id="INC-20260627-NODE110", sensors_succeeded=1, recent_logs="sanitized logs", mcp_health={"ssh_diagnose": True}, ) verified = await job.verify_ansible_candidate_pre_decision_evidence( snapshot=snapshot, project_id="awoooi", ) assert verified is True assert "CAST(mcp_health AS jsonb)" in statements[0] assert "CAST(post_execution_state AS jsonb) = 'null'::jsonb" in statements[0] @pytest.mark.asyncio async def test_backfill_skips_when_worker_disabled(monkeypatch: pytest.MonkeyPatch) -> None: from src.jobs import awooop_ansible_candidate_backfill_job as job monkeypatch.setattr(job.settings, "ENABLE_AWOOOP_ANSIBLE_CANDIDATE_BACKFILL_WORKER", False) result = await job.enqueue_missing_ansible_candidates_once() assert result["skipped"] is True assert result["queued"] == 0 def test_backfill_query_excludes_existing_ansible_candidate_rows() -> None: from src.jobs import awooop_ansible_candidate_backfill_job as job source = MagicMock(wraps=job._fetch_missing_candidate_incidents) query_source = job._fetch_missing_candidate_incidents.__code__.co_consts joined = "\n".join(str(item) for item in query_source) assert source is not None assert "NOT EXISTS" in joined assert "ansible_candidate_matched" in joined assert "resolved_at IS NULL" in joined def test_backfill_worker_error_backoff_is_short_and_bounded() -> None: from src.jobs import awooop_ansible_candidate_backfill_job as job delay = job._error_backoff_seconds() assert job._ERROR_BACKOFF_MIN_SECONDS <= delay <= job._ERROR_BACKOFF_MAX_SECONDS assert job._ERROR_BACKOFF_MAX_SECONDS < 60