fix(agent): replay repaired failed applies

apps/api/tests/test_awooop_ansible_post_verifier.py

@@ -69,7 +69,7 @@ def test_host110_node_exporter_postconditions_are_bounded() -> None:
     assert catalog["risk_level"] == "low"
     assert catalog["auto_apply_enabled"] is True
     assert catalog["catalog_revision"] == (
-        "2026-07-14-node-exporter-bounded-v4"
+        "2026-07-14-node-exporter-bounded-v5"
     )
 
 

apps/api/tests/test_host110_node_exporter_ansible_catalog.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+import inspect
+import subprocess
 from pathlib import Path
 
 import yaml
@@ -9,7 +11,10 @@ from src.services.awooop_ansible_audit_service import (
     get_ansible_catalog_item,
 )
 from src.services.awooop_ansible_check_mode_service import (
+    _claim_from_failed_apply_catalog_repair_row,
     _claim_from_stale_check_mode_row,
+    claim_catalog_drift_failed_applies,
+    run_pending_check_modes_once,
 )
 
 ROOT = Path(__file__).resolve().parents[3]
@@ -48,7 +53,7 @@ def test_node_exporter_playbook_only_reconciles_one_container() -> None:
     assert catalog is not None
     assert catalog["playbook_path"].endswith("110-node-exporter-repair.yml")
     assert catalog["catalog_revision"] == (
-        "2026-07-14-node-exporter-bounded-v4"
+        "2026-07-14-node-exporter-bounded-v5"
     )
     assert "docker\n          - compose\n          - up" in text
     assert "node-exporter" in text
@@ -63,6 +68,27 @@ def test_node_exporter_playbook_only_reconciles_one_container() -> None:
         assert forbidden not in text.lower()
 
 
+def test_node_exporter_post_apply_prometheus_probe_is_valid_bash() -> None:
+    payload = yaml.safe_load(PLAYBOOK.read_text(encoding="utf-8"))
+    verifier = next(
+        task
+        for task in payload[0]["tasks"]
+        if task["name"]
+        == "Verify Prometheus sees node-exporter after controlled apply"
+    )
+    command = verifier["ansible.builtin.shell"]["cmd"]
+
+    result = subprocess.run(
+        ["bash", "-n"],
+        input=command,
+        text=True,
+        capture_output=True,
+        check=False,
+    )
+
+    assert result.returncode == 0, result.stderr
+
+
 def test_existing_110_devops_lineage_migrates_to_bounded_revision() -> None:
     claim = _claim_from_stale_check_mode_row(
         {
@@ -90,6 +116,80 @@ def test_existing_110_devops_lineage_migrates_to_bounded_revision() -> None:
     assert claim.playbook_path.endswith("110-node-exporter-repair.yml")
     assert claim.apply_playbook_path.endswith("110-node-exporter-repair.yml")
     assert claim.input_payload["catalog_revision"] == (
-        "2026-07-14-node-exporter-bounded-v4"
+        "2026-07-14-node-exporter-bounded-v5"
     )
     assert claim.input_payload["controlled_apply_allowed"] is True
+
+
+def test_failed_apply_catalog_repair_reenters_same_automation_run() -> None:
+    failed_apply_op_id = "00000000-0000-0000-0000-000000000201"
+    source_candidate_op_id = "00000000-0000-0000-0000-000000000110"
+    claim = _claim_from_failed_apply_catalog_repair_row(
+        {
+            "op_id": failed_apply_op_id,
+            "incident_id": "INC-NODE-EXPORTER-110",
+            "input": {
+                "automation_run_id": source_candidate_op_id,
+                "source_candidate_op_id": source_candidate_op_id,
+                "incident_id": "INC-NODE-EXPORTER-110",
+                "catalog_id": "ansible:110-devops",
+                "catalog_revision": (
+                    "2026-07-14-node-exporter-bounded-v4"
+                ),
+                "playbook_path": (
+                    "infra/ansible/playbooks/110-node-exporter-repair.yml"
+                ),
+                "inventory_hosts": ["host_110"],
+                "risk_level": "low",
+            },
+        }
+    )
+
+    assert claim is not None
+    assert claim.source_candidate_op_id == source_candidate_op_id
+    assert claim.input_payload["automation_run_id"] == source_candidate_op_id
+    assert claim.input_payload["catalog_revision"] == (
+        "2026-07-14-node-exporter-bounded-v5"
+    )
+    assert claim.input_payload["catalog_repair_of_apply_op_id"] == (
+        failed_apply_op_id
+    )
+    assert claim.input_payload["controlled_apply_allowed"] is True
+    assert claim.input_payload["bounded_execution_scope"] == (
+        "catalog_repair_single_apply"
+    )
+
+
+def test_failed_apply_current_revision_is_not_replayed() -> None:
+    claim = _claim_from_failed_apply_catalog_repair_row(
+        {
+            "op_id": "00000000-0000-0000-0000-000000000201",
+            "incident_id": "INC-NODE-EXPORTER-110",
+            "input": {
+                "automation_run_id": (
+                    "00000000-0000-0000-0000-000000000110"
+                ),
+                "source_candidate_op_id": (
+                    "00000000-0000-0000-0000-000000000110"
+                ),
+                "incident_id": "INC-NODE-EXPORTER-110",
+                "catalog_id": "ansible:110-devops",
+                "catalog_revision": (
+                    "2026-07-14-node-exporter-bounded-v5"
+                ),
+            },
+        }
+    )
+
+    assert claim is None
+
+
+def test_failed_apply_catalog_repair_is_deduped_and_prioritized() -> None:
+    claim_source = inspect.getsource(claim_catalog_drift_failed_applies)
+    worker_source = inspect.getsource(run_pending_check_modes_once)
+
+    assert "catalog_repair_of_apply_op_id" in claim_source
+    assert "catalog_replay_revision" in claim_source
+    assert worker_source.index("claim_catalog_drift_failed_applies") < (
+        worker_source.index("claim_catalog_drift_failed_check_modes")
+    )