fix(agent): close verified alert lifecycle

apps/api/tests/test_ansible_verified_closure.py

@@ -0,0 +1,292 @@
+from __future__ import annotations
+
+from unittest.mock import AsyncMock
+
+import pytest
+
+from src.jobs import awooop_ansible_candidate_backfill_job as candidate_job
+from src.services import awooop_ansible_check_mode_service as service
+
+
+def _claim() -> service.AnsibleCheckModeClaim:
+    return service.AnsibleCheckModeClaim(
+        op_id="00000000-0000-0000-0000-000000000101",
+        source_candidate_op_id="00000000-0000-0000-0000-000000000100",
+        incident_id="INC-20260711-D037E5",
+        catalog_id="ansible:awoooi-auto-repair-canary",
+        playbook_path="infra/ansible/playbooks/awoooi-auto-repair-canary.yml",
+        apply_playbook_path=(
+            "infra/ansible/playbooks/awoooi-auto-repair-canary.yml"
+        ),
+        inventory_hosts=("host_121",),
+        risk_level="medium",
+        input_payload={
+            "automation_run_id": "00000000-0000-0000-0000-000000000102",
+            "approval_id": "00000000-0000-0000-0000-000000000103",
+        },
+    )
+
+
+def _verified_result() -> service.AnsibleRunResult:
+    return service.AnsibleRunResult(
+        returncode=0,
+        stdout="",
+        stderr="",
+        duration_ms=25,
+        post_verifier_passed=True,
+    )
+
+
+def test_runtime_stage_ids_only_accepts_durable_receipts() -> None:
+    stage_ids = service._runtime_stage_ids(
+        {
+            "runtime_stage_receipts": [
+                {
+                    "stage_id": "mcp_context",
+                    "durable_receipt": True,
+                },
+                {
+                    "stage_id": "telegram_receipt",
+                    "durable_receipt": False,
+                },
+            ]
+        }
+    )
+
+    assert stage_ids == {"mcp_context"}
+
+
+@pytest.mark.asyncio
+async def test_closure_refuses_to_resolve_when_any_receipt_is_missing(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    terminal_writer = AsyncMock()
+    monkeypatch.setattr(
+        service,
+        "_read_verified_apply_closure_prerequisites",
+        AsyncMock(
+            return_value={
+                "ready": False,
+                "missing": ["telegram_receipt"],
+                "receipts": {},
+            }
+        ),
+    )
+    monkeypatch.setattr(
+        service,
+        "_record_incident_terminal_disposition",
+        terminal_writer,
+    )
+
+    result = await service._finalize_verified_apply_closure(
+        _claim(),
+        apply_op_id="00000000-0000-0000-0000-000000000104",
+        project_id="awoooi",
+    )
+
+    assert result == {
+        "status": "closure_receipts_pending",
+        "closed": False,
+        "missing": ["telegram_receipt"],
+    }
+    terminal_writer.assert_not_awaited()
+
+
+@pytest.mark.asyncio
+async def test_closure_resolves_only_after_durable_readback(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.setattr(
+        service,
+        "_read_verified_apply_closure_prerequisites",
+        AsyncMock(return_value={"ready": True, "receipts": {"all": True}}),
+    )
+    monkeypatch.setattr(
+        service,
+        "_record_incident_terminal_disposition",
+        AsyncMock(
+            return_value={
+                "automation_run_id": (
+                    "00000000-0000-0000-0000-000000000102"
+                ),
+                "apply_op_id": "00000000-0000-0000-0000-000000000104",
+                "incident_resolved": True,
+            }
+        ),
+    )
+    receipt_writer = AsyncMock(return_value=True)
+    lifecycle_writer = AsyncMock(return_value=True)
+    monkeypatch.setattr(
+        service,
+        "_append_runtime_stage_receipts_to_apply",
+        receipt_writer,
+    )
+    monkeypatch.setattr(
+        service,
+        "_append_alert_lifecycle_receipt",
+        lifecycle_writer,
+    )
+    monkeypatch.setattr(
+        service,
+        "_read_incident_closure_readback",
+        AsyncMock(
+            return_value={
+                "closed": True,
+                "missing": [],
+                "incident_status": "RESOLVED",
+            }
+        ),
+    )
+
+    result = await service._finalize_verified_apply_closure(
+        _claim(),
+        apply_op_id="00000000-0000-0000-0000-000000000104",
+        project_id="awoooi",
+    )
+
+    assert result["status"] == "controlled_apply_closed"
+    assert result["closed"] is True
+    closure_receipt = receipt_writer.await_args.kwargs["receipts"][0]
+    assert closure_receipt["stage_id"] == "incident_closure"
+    assert closure_receipt["durable_receipt"] is True
+    assert lifecycle_writer.await_args.args[1] == "RESOLVED"
+
+
+@pytest.mark.asyncio
+async def test_projection_replay_sends_missing_receipt_without_reapplying(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    readback = AsyncMock(
+        side_effect=[
+            {"receipts": {"telegram_receipt": False}},
+            {"receipts": {"telegram_receipt": True}},
+        ]
+    )
+    telegram_sender = AsyncMock(return_value={"ok": True})
+    monkeypatch.setattr(
+        service,
+        "_finalize_controlled_approval_projection",
+        AsyncMock(return_value=True),
+    )
+    monkeypatch.setattr(
+        service,
+        "_append_alert_lifecycle_receipt",
+        AsyncMock(return_value=True),
+    )
+    monkeypatch.setattr(
+        service,
+        "_read_verified_apply_closure_prerequisites",
+        readback,
+    )
+    monkeypatch.setattr(
+        service,
+        "_send_controlled_apply_telegram_receipt",
+        telegram_sender,
+    )
+    monkeypatch.setattr(
+        service,
+        "_finalize_verified_apply_closure",
+        AsyncMock(return_value={"status": "controlled_apply_closed", "closed": True}),
+    )
+
+    result = await service._reconcile_verified_apply_closure_projections(
+        _claim(),
+        _verified_result(),
+        apply_op_id="00000000-0000-0000-0000-000000000104",
+        writeback={
+            "verification_passed": True,
+            "verification_result": "success",
+            "verification": True,
+            "learning": True,
+        },
+        project_id="awoooi",
+    )
+
+    assert result["closed"] is True
+    assert result["runtime_apply_executed"] is False
+    telegram_sender.assert_awaited_once()
+
+
+def test_backfill_preserves_approval_and_replaces_terminal_skipped_candidate() -> None:
+    proposal = candidate_job._build_backfill_proposal(
+        {
+            "alertname": "AwoooPAutoRepairCanaryT16",
+            "severity": "medium",
+            "approval_id": "00000000-0000-0000-0000-000000000103",
+        }
+    )
+    query_source = "\n".join(
+        str(value)
+        for value in candidate_job._fetch_missing_candidate_incidents.__code__.co_consts
+    )
+
+    assert proposal["approval_id"] == "00000000-0000-0000-0000-000000000103"
+    assert "approval_records" in query_source
+    assert "ansible_execution_skipped" in query_source
+    assert "terminal.parent_op_id = existing.op_id" in query_source
+
+
+@pytest.mark.asyncio
+async def test_broker_reconciles_receipts_before_claiming_fresh_work(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    retry_replayer = AsyncMock()
+    monkeypatch.setattr(
+        service,
+        "_expire_stale_ansible_execution_capabilities",
+        AsyncMock(return_value=0),
+    )
+    monkeypatch.setattr(
+        service,
+        "backfill_missing_auto_repair_execution_receipts_once",
+        AsyncMock(
+            return_value={
+                "scanned": 1,
+                "written": 1,
+                "incident_closure_written": 1,
+                "telegram_receipt_acknowledged": 1,
+                "error": None,
+            }
+        ),
+    )
+    monkeypatch.setattr(
+        service,
+        "run_failed_apply_check_mode_replay_once",
+        retry_replayer,
+    )
+
+    result = await service.run_pending_check_modes_once(limit=1)
+
+    assert result["claimed"] == 0
+    assert result["repair_receipt_backfill_priority_tick"] is True
+    assert result["repair_receipt_closure_written"] == 1
+    retry_replayer.assert_not_awaited()
+
+
+@pytest.mark.asyncio
+async def test_signal_worker_retry_preflight_is_query_only(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.setattr(
+        service,
+        "_load_open_failed_apply_retry_row",
+        AsyncMock(return_value={"op_id": "apply-op"}),
+    )
+
+    result = await service.preflight_failed_apply_retry_queue_once(
+        project_id="awoooi",
+        window_hours=24,
+    )
+
+    assert result["scanned"] == 1
+    assert result["replayed"] == 0
+    assert result["query_only"] is True
+    assert result["runtime_apply_executed"] is False
+    assert result["execution_owner"] == "awoooi-ansible-executor-broker"
+
+
+def test_candidate_worker_defaults_to_query_only_retry_preflight() -> None:
+    defaults = candidate_job.enqueue_missing_ansible_candidates_once.__kwdefaults__
+
+    assert defaults is not None
+    assert defaults["retry_replayer"] is service.preflight_failed_apply_retry_queue_once

apps/api/tests/test_awooop_truth_chain_service.py

@@ -5,6 +5,7 @@ import os
 from datetime import UTC, datetime, timedelta
 from pathlib import Path
 from types import SimpleNamespace
+from unittest.mock import AsyncMock
 
 import pytest
 
@@ -2071,6 +2072,11 @@ async def test_catalog_drift_query_failure_does_not_block_fresh_candidate_claims
         "_expire_stale_ansible_execution_capabilities",
         no_expired_capabilities,
     )
+    monkeypatch.setattr(
+        service,
+        "backfill_missing_auto_repair_execution_receipts_once",
+        AsyncMock(return_value={"scanned": 0, "written": 0, "error": None}),
+    )
     monkeypatch.setattr(service, "_runtime_blockers", lambda: [])
     monkeypatch.setattr(
         service,
@@ -2133,6 +2139,11 @@ async def test_execution_broker_runs_failed_apply_retry_before_all_candidate_cla
         "_expire_stale_ansible_execution_capabilities",
         no_expired_capabilities,
     )
+    monkeypatch.setattr(
+        service,
+        "backfill_missing_auto_repair_execution_receipts_once",
+        AsyncMock(return_value={"scanned": 0, "written": 0, "error": None}),
+    )
     monkeypatch.setattr(service, "_runtime_blockers", lambda: [])
     monkeypatch.setattr(
         service,