feat(agent): execute failed apply check-mode replay

apps/api/tests/test_awooop_ansible_candidate_backfill_job.py

@@ -66,6 +66,15 @@ async def test_backfill_enqueues_catalog_matched_incident(monkeypatch: pytest.Mo
         assert kwargs["project_id"] == "awoooi"
         return {"written": 2, "error": None}
 
+    async def fake_retry_replayer(**kwargs):
+        assert kwargs["project_id"] == "awoooi"
+        return {
+            "replayed": 1,
+            "check_mode_passed": 1,
+            "check_mode_failed": 0,
+            "error": None,
+        }
+
     monkeypatch.setattr(job, "get_db_context", fake_db_context)
     monkeypatch.setattr(job.settings, "ENABLE_AWOOOP_ANSIBLE_CANDIDATE_BACKFILL_WORKER", True)
 
@@ -75,11 +84,15 @@ async def test_backfill_enqueues_catalog_matched_incident(monkeypatch: pytest.Mo
         window_hours=24,
         recorder=fake_recorder,
         receipt_backfiller=fake_receipt_backfiller,
+        retry_replayer=fake_retry_replayer,
     )
 
     assert result["queued"] == 1
     assert result["no_catalog_candidate"] == 0
     assert result["repair_receipts_backfilled"] == 2
+    assert result["failed_apply_retry_replayed"] == 1
+    assert result["failed_apply_retry_check_mode_passed"] == 1
+    assert result["failed_apply_retry_check_mode_failed"] == 0
     assert recorded[0]["decision_path"] == "repair_candidate_controlled_queue"
     assert recorded[0]["incident"]["incident_id"] == "INC-20260627-NODE110"
 

apps/api/tests/test_awooop_truth_chain_service.py

@@ -35,6 +35,7 @@ from src.services.awooop_ansible_check_mode_service import (
     finalize_check_mode_claim,
     recent_ansible_transport_blockers,
     run_controlled_apply_for_claim,
+    run_failed_apply_check_mode_replay_once,
     run_pending_check_modes_once,
 )
 from src.services.awooop_truth_chain_service import (
@@ -1778,6 +1779,19 @@ def test_ansible_live_controlled_apply_sends_telegram_receipt_but_backfill_does_
     assert inspect.iscoroutinefunction(_send_controlled_apply_telegram_receipt)
 
 
+def test_failed_apply_retry_replay_is_no_write_and_idempotent() -> None:
+    source = inspect.getsource(run_failed_apply_check_mode_replay_once)
+
+    assert "FOR UPDATE SKIP LOCKED" in source
+    assert "controlled_retry_check_mode_replay" in source
+    assert "build_ansible_check_mode_command" in source
+    assert "controlled_apply_allowed=False" in source
+    assert '"runtime_apply_executed": False' in source
+    assert "retry_requires_repair_and_new_apply_gate" in source
+    assert "NOT EXISTS" in source
+    assert "run_controlled_apply_for_claim" not in source
+
+
 def test_ansible_check_and_apply_rows_persist_canonical_automation_run_id() -> None:
     finalize_source = inspect.getsource(finalize_check_mode_claim)
     apply_source = inspect.getsource(run_controlled_apply_for_claim)