316 lines
10 KiB
Python
316 lines
10 KiB
Python
from __future__ import annotations
|
|
|
|
from contextlib import asynccontextmanager
|
|
from unittest.mock import AsyncMock, MagicMock
|
|
|
|
import pytest
|
|
|
|
|
|
class _FakeMappings:
|
|
def __init__(self, rows: list[dict]):
|
|
self._rows = rows
|
|
|
|
def all(self) -> list[dict]:
|
|
return self._rows
|
|
|
|
|
|
class _FakeResult:
|
|
def __init__(self, rows: list[dict]):
|
|
self._rows = rows
|
|
|
|
def mappings(self) -> _FakeMappings:
|
|
return _FakeMappings(self._rows)
|
|
|
|
|
|
def _candidate_incident() -> dict:
|
|
return {
|
|
"incident_id": "INC-20260627-NODE110",
|
|
"project_id": "awoooi",
|
|
"status": "INVESTIGATING",
|
|
"severity": "warning",
|
|
"alertname": "NodeExporterDown",
|
|
"alert_category": "infrastructure",
|
|
"notification_type": "TYPE-3",
|
|
"signals": [
|
|
{
|
|
"alert_name": "NodeExporterDown",
|
|
"labels": {
|
|
"alertname": "NodeExporterDown",
|
|
"instance": "node-exporter-110",
|
|
},
|
|
"annotations": {},
|
|
}
|
|
],
|
|
}
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_backfill_enqueues_catalog_matched_incident(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
from src.jobs import awooop_ansible_candidate_backfill_job as job
|
|
|
|
fake_db = AsyncMock()
|
|
fake_db.execute = AsyncMock(return_value=_FakeResult([_candidate_incident()]))
|
|
|
|
@asynccontextmanager
|
|
async def fake_db_context(project_id: str = "awoooi"):
|
|
assert project_id == "awoooi"
|
|
yield fake_db
|
|
|
|
recorded: list[dict] = []
|
|
call_order: list[str] = []
|
|
|
|
async def fake_evidence_collector(**kwargs):
|
|
call_order.append("evidence")
|
|
assert kwargs["project_id"] == "awoooi"
|
|
assert kwargs["automation_run_id"]
|
|
return MagicMock(
|
|
snapshot_id="evidence-1",
|
|
incident_id="INC-20260627-NODE110",
|
|
sensors_succeeded=1,
|
|
recent_logs="sanitized logs",
|
|
)
|
|
|
|
async def fake_evidence_verifier(**kwargs):
|
|
call_order.append("verify")
|
|
assert kwargs["snapshot"].snapshot_id == "evidence-1"
|
|
return True
|
|
|
|
async def fake_recorder(**kwargs):
|
|
call_order.append("record")
|
|
recorded.append(kwargs)
|
|
return True
|
|
|
|
async def fake_receipt_backfiller(**kwargs):
|
|
call_order.append("receipt")
|
|
assert kwargs["project_id"] == "awoooi"
|
|
return {"written": 2, "error": None}
|
|
|
|
async def fake_retry_replayer(**kwargs):
|
|
call_order.append("retry")
|
|
assert kwargs["project_id"] == "awoooi"
|
|
return {
|
|
"scanned": 0,
|
|
"replayed": 0,
|
|
"check_mode_passed": 0,
|
|
"check_mode_failed": 0,
|
|
"runtime_stage_receipt_written": 0,
|
|
"blockers": [],
|
|
"error": None,
|
|
}
|
|
|
|
monkeypatch.setattr(job, "get_db_context", fake_db_context)
|
|
monkeypatch.setattr(job.settings, "ENABLE_AWOOOP_ANSIBLE_CANDIDATE_BACKFILL_WORKER", True)
|
|
|
|
result = await job.enqueue_missing_ansible_candidates_once(
|
|
project_id="awoooi",
|
|
limit=5,
|
|
window_hours=24,
|
|
recorder=fake_recorder,
|
|
receipt_backfiller=fake_receipt_backfiller,
|
|
retry_replayer=fake_retry_replayer,
|
|
evidence_collector=fake_evidence_collector,
|
|
evidence_verifier=fake_evidence_verifier,
|
|
)
|
|
|
|
assert result["queued"] == 1
|
|
assert result["no_catalog_candidate"] == 0
|
|
assert result["repair_receipts_backfilled"] == 2
|
|
assert result["failed_apply_retry_replayed"] == 0
|
|
assert result["failed_apply_retry_check_mode_passed"] == 0
|
|
assert result["failed_apply_retry_check_mode_failed"] == 0
|
|
assert result["failed_apply_retry_stage_receipt_written"] == 0
|
|
assert result["failed_apply_retry_blockers"] == []
|
|
assert result["failed_apply_retry_priority_tick"] is False
|
|
assert result["pre_decision_evidence_ready"] == 1
|
|
assert result["pre_decision_evidence_blocked"] == 0
|
|
assert call_order == ["retry", "receipt", "evidence", "verify", "record"]
|
|
assert recorded[0]["decision_path"] == "repair_candidate_controlled_queue"
|
|
assert recorded[0]["incident"]["incident_id"] == "INC-20260627-NODE110"
|
|
assert recorded[0]["automation_run_id"]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_backfill_runs_retry_then_continues_fresh_candidates(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
from src.jobs import awooop_ansible_candidate_backfill_job as job
|
|
|
|
async def fake_retry_replayer(**kwargs):
|
|
assert kwargs["project_id"] == "awoooi"
|
|
return {
|
|
"scanned": 1,
|
|
"replayed": 1,
|
|
"check_mode_passed": 1,
|
|
"check_mode_failed": 0,
|
|
"runtime_stage_receipt_written": 1,
|
|
"blockers": [],
|
|
"error": None,
|
|
}
|
|
|
|
fresh_scan = AsyncMock(return_value=[])
|
|
|
|
monkeypatch.setattr(job.settings, "ENABLE_AWOOOP_ANSIBLE_CANDIDATE_BACKFILL_WORKER", True)
|
|
monkeypatch.setattr(job, "_fetch_missing_candidate_incidents", fresh_scan)
|
|
receipt_backfiller = AsyncMock(return_value={"written": 0, "error": None})
|
|
recorder = AsyncMock(return_value=True)
|
|
|
|
result = await job.enqueue_missing_ansible_candidates_once(
|
|
retry_replayer=fake_retry_replayer,
|
|
receipt_backfiller=receipt_backfiller,
|
|
recorder=recorder,
|
|
)
|
|
|
|
assert result["failed_apply_retry_scanned"] == 1
|
|
assert result["failed_apply_retry_replayed"] == 1
|
|
assert result["failed_apply_retry_check_mode_passed"] == 1
|
|
assert result["failed_apply_retry_stage_receipt_written"] == 1
|
|
assert result["failed_apply_retry_priority_tick"] is True
|
|
assert result["queued"] == 0
|
|
receipt_backfiller.assert_awaited_once()
|
|
fresh_scan.assert_awaited_once()
|
|
recorder.assert_not_awaited()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_backfill_surfaces_retry_blockers_and_continues_fresh_scan(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
from src.jobs import awooop_ansible_candidate_backfill_job as job
|
|
|
|
async def fake_retry_replayer(**_kwargs):
|
|
return {
|
|
"scanned": 0,
|
|
"replayed": 0,
|
|
"blockers": ["database_connection_budget_exhausted"],
|
|
"error": None,
|
|
}
|
|
|
|
fresh_scan = AsyncMock(return_value=[])
|
|
|
|
monkeypatch.setattr(job.settings, "ENABLE_AWOOOP_ANSIBLE_CANDIDATE_BACKFILL_WORKER", True)
|
|
monkeypatch.setattr(job, "_fetch_missing_candidate_incidents", fresh_scan)
|
|
|
|
result = await job.enqueue_missing_ansible_candidates_once(
|
|
retry_replayer=fake_retry_replayer,
|
|
receipt_backfiller=AsyncMock(return_value={"written": 0, "error": None}),
|
|
)
|
|
|
|
assert result["failed_apply_retry_blockers"] == [
|
|
"database_connection_budget_exhausted"
|
|
]
|
|
assert result["failed_apply_retry_priority_tick"] is True
|
|
assert result["queued"] == 0
|
|
fresh_scan.assert_awaited_once()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_backfill_retries_automatically_when_predecision_evidence_is_incomplete(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
from src.jobs import awooop_ansible_candidate_backfill_job as job
|
|
|
|
fake_db = AsyncMock()
|
|
fake_db.execute = AsyncMock(return_value=_FakeResult([_candidate_incident()]))
|
|
|
|
@asynccontextmanager
|
|
async def fake_db_context(project_id: str = "awoooi"):
|
|
yield fake_db
|
|
|
|
async def fake_evidence_collector(**_kwargs):
|
|
return None
|
|
|
|
async def fake_evidence_verifier(**kwargs):
|
|
assert kwargs["snapshot"] is None
|
|
return False
|
|
|
|
recorder = AsyncMock(return_value=True)
|
|
monkeypatch.setattr(job, "get_db_context", fake_db_context)
|
|
monkeypatch.setattr(job.settings, "ENABLE_AWOOOP_ANSIBLE_CANDIDATE_BACKFILL_WORKER", True)
|
|
|
|
result = await job.enqueue_missing_ansible_candidates_once(
|
|
recorder=recorder,
|
|
evidence_collector=fake_evidence_collector,
|
|
evidence_verifier=fake_evidence_verifier,
|
|
receipt_backfiller=AsyncMock(return_value={"written": 0, "error": None}),
|
|
retry_replayer=AsyncMock(return_value={"error": None}),
|
|
)
|
|
|
|
assert result["queued"] == 0
|
|
assert result["pre_decision_evidence_blocked"] == 1
|
|
recorder.assert_not_awaited()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_predecision_verifier_casts_json_health_to_jsonb(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
from src.jobs import awooop_ansible_candidate_backfill_job as job
|
|
|
|
statements: list[str] = []
|
|
|
|
class ScalarResult:
|
|
def scalar(self):
|
|
return True
|
|
|
|
class FakeDb:
|
|
async def execute(self, statement, _parameters):
|
|
statements.append(str(statement))
|
|
return ScalarResult()
|
|
|
|
@asynccontextmanager
|
|
async def fake_db_context(project_id: str = "awoooi"):
|
|
assert project_id == "awoooi"
|
|
yield FakeDb()
|
|
|
|
monkeypatch.setattr(job, "get_db_context", fake_db_context)
|
|
snapshot = MagicMock(
|
|
snapshot_id="evidence-1",
|
|
incident_id="INC-20260627-NODE110",
|
|
sensors_succeeded=1,
|
|
recent_logs="sanitized logs",
|
|
mcp_health={"ssh_diagnose": True},
|
|
)
|
|
|
|
verified = await job.verify_ansible_candidate_pre_decision_evidence(
|
|
snapshot=snapshot,
|
|
project_id="awoooi",
|
|
)
|
|
|
|
assert verified is True
|
|
assert "CAST(mcp_health AS jsonb)" in statements[0]
|
|
assert "CAST(post_execution_state AS jsonb) = 'null'::jsonb" in statements[0]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_backfill_skips_when_worker_disabled(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
from src.jobs import awooop_ansible_candidate_backfill_job as job
|
|
|
|
monkeypatch.setattr(job.settings, "ENABLE_AWOOOP_ANSIBLE_CANDIDATE_BACKFILL_WORKER", False)
|
|
|
|
result = await job.enqueue_missing_ansible_candidates_once()
|
|
|
|
assert result["skipped"] is True
|
|
assert result["queued"] == 0
|
|
|
|
|
|
def test_backfill_query_excludes_existing_ansible_candidate_rows() -> None:
|
|
from src.jobs import awooop_ansible_candidate_backfill_job as job
|
|
|
|
source = MagicMock(wraps=job._fetch_missing_candidate_incidents)
|
|
query_source = job._fetch_missing_candidate_incidents.__code__.co_consts
|
|
joined = "\n".join(str(item) for item in query_source)
|
|
|
|
assert source is not None
|
|
assert "NOT EXISTS" in joined
|
|
assert "ansible_candidate_matched" in joined
|
|
assert "resolved_at IS NULL" in joined
|
|
|
|
|
|
def test_backfill_worker_error_backoff_is_short_and_bounded() -> None:
|
|
from src.jobs import awooop_ansible_candidate_backfill_job as job
|
|
|
|
delay = job._error_backoff_seconds()
|
|
|
|
assert job._ERROR_BACKOFF_MIN_SECONDS <= delay <= job._ERROR_BACKOFF_MAX_SECONDS
|
|
assert job._ERROR_BACKOFF_MAX_SECONDS < 60
|