fix(agent): route disk pressure to bounded repair

apps/api/tests/test_ansible_verified_closure.py

@@ -992,6 +992,63 @@ async def test_historical_no_write_backfill_is_provider_silent(
     assert policy["suppression_reason"] == "historical_projection_backfill"
 
 
+@pytest.mark.asyncio
+async def test_historical_verified_repair_is_provider_silent(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    db = _SequenceDB(
+        _MappingResult(),
+        _MappingResult(row=None),
+    )
+
+    @asynccontextmanager
+    async def fake_get_db_context(_project_id):
+        yield db
+
+    record_outbound = AsyncMock(
+        return_value="00000000-0000-0000-0000-000000000498"
+    )
+    monkeypatch.setattr("src.db.base.get_db_context", fake_get_db_context)
+    monkeypatch.setattr(
+        "src.services.channel_hub.record_outbound_message",
+        record_outbound,
+    )
+    gateway = TelegramGateway()
+    identity = {
+        "project_id": "awoooi",
+        "automation_run_id": "00000000-0000-0000-0000-000000000402",
+        "incident_id": "INC-20260711-HISTORICAL-SUCCESS",
+        "apply_op_id": "00000000-0000-0000-0000-000000000404",
+    }
+    source_extra = {
+        "callback_reply": {
+            "action": "controlled_apply_result",
+            "execution_kind": "controlled_apply",
+            **identity,
+        },
+        "notification_policy": {
+            "failure_fingerprint": "",
+            "disposition": "suppressed",
+            "provider_delivery": "shadow_only",
+        },
+    }
+
+    reservation = await gateway._reserve_controlled_apply_result_outbound(
+        identity=identity,
+        payload={"chat_id": "sre-chat", "text": "historical repair closed"},
+        source_envelope_extra=source_extra,
+    )
+
+    assert reservation["status"] == "suppressed"
+    assert len(db.statements) == 2
+    record_kwargs = record_outbound.await_args.kwargs
+    assert record_kwargs["send_status"] == "shadow"
+    assert record_kwargs["provider_message_id"] is None
+    policy = record_kwargs["source_envelope"]["notification_policy"]
+    assert policy["suppression_reason"] == "historical_projection_backfill"
+    assert policy["provider_send_performed"] is False
+
+
 def test_controlled_result_uses_durable_reservation_and_single_reconcile_entry() -> None:
     reserve_source = inspect.getsource(
         TelegramGateway._reserve_controlled_apply_result_outbound

apps/api/tests/test_host_disk_pressure_ansible_catalog.py

@@ -0,0 +1,265 @@
+from __future__ import annotations
+
+import inspect
+from contextlib import asynccontextmanager
+from pathlib import Path
+from uuid import UUID
+
+import pytest
+import yaml
+
+from src.services.awooop_ansible_audit_service import (
+    _catalog_hints,
+    get_ansible_catalog_item,
+)
+from src.services.awooop_ansible_check_mode_service import (
+    _claim_from_stale_check_mode_row,
+    _send_controlled_apply_telegram_receipt,
+    claim_catalog_drift_failed_check_modes,
+    claim_semantically_misrouted_verified_applies,
+    run_pending_check_modes_once,
+)
+from src.services.awooop_ansible_post_verifier import postconditions_for_catalog
+
+ROOT = Path(__file__).resolve().parents[3]
+PLAYBOOK = (
+    ROOT
+    / "infra"
+    / "ansible"
+    / "playbooks"
+    / "host-disk-pressure-bounded-cleanup.yml"
+)
+
+
+def _disk_incident(host: str) -> dict:
+    return {
+        "alertname": "HostOutOfDiskSpace",
+        "alert_category": "host_resource",
+        "affected_services": [f"node-exporter-{host}"],
+    }
+
+
+def test_disk_alert_routes_to_the_matching_host_catalog() -> None:
+    host110 = _catalog_hints(_disk_incident("110"), None)
+    host188 = _catalog_hints(_disk_incident("188"), None)
+
+    assert [row["catalog_id"] for row in host110["candidates"]] == [
+        "ansible:110-disk-pressure"
+    ]
+    assert [row["catalog_id"] for row in host188["candidates"]] == [
+        "ansible:188-disk-pressure"
+    ]
+    assert all(
+        row["catalog_id"] != "ansible:110-devops"
+        for row in [*host110["candidates"], *host188["candidates"]]
+    )
+
+
+def test_node_exporter_repair_requires_a_down_signal_not_a_disk_alert() -> None:
+    disk_hints = _catalog_hints(_disk_incident("110"), None)
+    down_hints = _catalog_hints(
+        {
+            "alertname": "NodeExporterDown",
+            "affected_services": ["node-exporter-110"],
+        },
+        None,
+    )
+
+    assert "ansible:110-devops" not in {
+        row["catalog_id"] for row in disk_hints["candidates"]
+    }
+    assert [row["catalog_id"] for row in down_hints["candidates"]] == [
+        "ansible:110-devops"
+    ]
+
+
+def test_disk_cleanup_playbook_excludes_destructive_asset_classes() -> None:
+    payload = yaml.safe_load(PLAYBOOK.read_text(encoding="utf-8"))
+    source = PLAYBOOK.read_text(encoding="utf-8").lower()
+
+    assert payload[0]["hosts"] == "devops:ai_web"
+    assert payload[0]["become"] is False
+    assert "docker-disk-pressure-retention-cleanup.py" in source
+    assert "--apply" in source
+    assert "--max-image-removals" in source
+    assert "single_apply_image_removal_is_count_bounded" in source
+    assert "--include-builder-cache" not in source
+    for forbidden in (
+        "docker system prune",
+        "docker volume prune",
+        "truncate -s",
+        "rm -rf",
+        "reboot",
+        "systemctl restart",
+    ):
+        assert forbidden not in source
+
+
+def test_stale_generic_host188_row_is_semantically_remapped() -> None:
+    claim = _claim_from_stale_check_mode_row(
+        {
+            "op_id": "00000000-0000-0000-0000-000000000188",
+            "parent_op_id": "00000000-0000-0000-0000-000000000100",
+            "incident_id": "INC-DISK-188",
+            "incident_alertname": "HostOutOfDiskSpace",
+            "incident_alert_category": "host_resource",
+            "incident_affected_services": ["node-exporter-188"],
+            "input": {
+                "source_candidate_op_id": (
+                    "00000000-0000-0000-0000-000000000100"
+                ),
+                "incident_id": "INC-DISK-188",
+                "catalog_id": "ansible:110-devops",
+                "catalog_playbook_path": "infra/ansible/playbooks/110-devops.yml",
+                "playbook_path": "infra/ansible/playbooks/110-devops.yml",
+                "inventory_hosts": ["host_110"],
+                "risk_level": "medium",
+            },
+        }
+    )
+
+    assert claim is not None
+    assert claim.catalog_id == "ansible:188-disk-pressure"
+    assert claim.inventory_hosts == ("host_188",)
+    assert claim.input_payload["catalog_route_changed"] is True
+    assert claim.input_payload["incident_semantic_route_rechecked"] is True
+
+
+def test_disk_catalogs_have_independent_threshold_verifiers() -> None:
+    for host in ("110", "188"):
+        catalog_id = f"ansible:{host}-disk-pressure"
+        catalog = get_ansible_catalog_item(catalog_id)
+        conditions = postconditions_for_catalog(catalog_id)
+
+        assert catalog is not None
+        assert catalog["auto_apply_enabled"] is True
+        assert catalog["risk_level"] == "medium"
+        assert len(conditions) == 2
+        assert any("df -P /" in condition.probe for condition in conditions)
+        assert any("docker info" in condition.probe for condition in conditions)
+
+
+def test_historical_catalog_replay_is_per_failed_row_and_provider_silent() -> None:
+    claim_source = inspect.getsource(claim_catalog_drift_failed_check_modes)
+    send_source = inspect.getsource(_send_controlled_apply_telegram_receipt)
+
+    assert "replay_of_check_mode_op_id" in claim_source
+    assert "failed_check_mode_op_id" in claim_source
+    assert '"historical_projection_backfill": True' in claim_source
+    assert '"telegram_provider_delivery": "shadow_only"' in claim_source
+    assert "telegram_provider_delivery" in send_source
+
+
+def test_verified_wrong_catalog_is_reconciled_before_failure_replays() -> None:
+    claim_source = inspect.getsource(
+        claim_semantically_misrouted_verified_applies
+    )
+    worker_source = inspect.getsource(run_pending_check_modes_once)
+
+    assert "semantic_route_reconciliation" in claim_source
+    assert "semantic_repair_of_apply_op_id" in claim_source
+    assert '"telegram_provider_delivery": "shadow_only"' in claim_source
+    assert "independent_post_verifier_passed" in claim_source
+    assert "seen_incident_ids" in claim_source
+    assert "RESOLVED" not in claim_source
+    assert worker_source.index(
+        "claim_semantically_misrouted_verified_applies"
+    ) < worker_source.index("claim_catalog_drift_failed_applies")
+    assert "semantic_catalog_reconciled" in worker_source
+
+
+@pytest.mark.asyncio
+async def test_verified_wrong_catalog_enqueues_one_shadow_disk_repair(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    from src.services import awooop_ansible_check_mode_service as service
+
+    previous_apply_op_id = "00000000-0000-0000-0000-000000000202"
+
+    class Result:
+        def __init__(self, *, rows=None, scalar=None):
+            self._rows = rows or []
+            self._scalar = scalar
+
+        def mappings(self):
+            return self
+
+        def all(self):
+            return self._rows
+
+        def scalar_one_or_none(self):
+            return self._scalar
+
+    class DB:
+        def __init__(self):
+            self.statements: list[str] = []
+
+        async def execute(self, statement, _params):
+            self.statements.append(str(statement))
+            if len(self.statements) == 1:
+                return Result(
+                    rows=[
+                        {
+                            "op_id": (
+                                "00000000-0000-0000-0000-000000000201"
+                            ),
+                            "parent_op_id": (
+                                "00000000-0000-0000-0000-000000000200"
+                            ),
+                            "incident_id": "INC-DISK-188-VERIFIED-WRONG",
+                            "incident_alertname": "HostOutOfDiskSpace",
+                            "incident_alert_category": "host_resource",
+                            "incident_affected_services": [
+                                "node-exporter-188"
+                            ],
+                            "previous_apply_op_id": previous_apply_op_id,
+                            "input": {
+                                "source_candidate_op_id": (
+                                    "00000000-0000-0000-0000-000000000200"
+                                ),
+                                "incident_id": (
+                                    "INC-DISK-188-VERIFIED-WRONG"
+                                ),
+                                "catalog_id": "ansible:110-devops",
+                                "catalog_playbook_path": (
+                                    "infra/ansible/playbooks/"
+                                    "110-node-exporter-repair.yml"
+                                ),
+                                "playbook_path": (
+                                    "infra/ansible/playbooks/"
+                                    "110-node-exporter-repair.yml"
+                                ),
+                                "inventory_hosts": ["host_110"],
+                                "risk_level": "low",
+                            },
+                        }
+                    ]
+                )
+            return Result(
+                scalar="00000000-0000-0000-0000-000000000203"
+            )
+
+    db = DB()
+
+    @asynccontextmanager
+    async def fake_get_db_context(_project_id):
+        yield db
+
+    monkeypatch.setattr(service, "get_db_context", fake_get_db_context)
+
+    claims = await claim_semantically_misrouted_verified_applies(limit=1)
+
+    assert len(claims) == 1
+    claim = claims[0]
+    assert claim.catalog_id == "ansible:188-disk-pressure"
+    assert claim.inventory_hosts == ("host_188",)
+    assert claim.input_payload["semantic_repair_of_apply_op_id"] == (
+        previous_apply_op_id
+    )
+    assert UUID(claim.input_payload["automation_run_id"])
+    assert claim.input_payload["automation_run_id"] != (
+        "00000000-0000-0000-0000-000000000200"
+    )
+    assert claim.input_payload["telegram_provider_delivery"] == "shadow_only"
+    assert "DISTINCT ON" not in db.statements[0]
+    assert "FOR UPDATE OF apply SKIP LOCKED" in db.statements[0]