awoooi/apps/api/tests/test_awooop_ansible_post_verifier.py

from __future__ import annotations

import inspect
from pathlib import Path

import pytest

from src.services.awooop_ansible_audit_service import list_ansible_catalog
from src.services.awooop_ansible_check_mode_service import (
    AnsibleRunResult,
    _post_apply_verification_result,
    _record_apply_post_verifier_terminal,
    run_pending_check_modes_once,
)
from src.services.awooop_ansible_post_verifier import (
    AssetPostcondition,
    ReadOnlyProbeResult,
    build_read_only_probe_command,
    registered_catalog_ids,
    run_ansible_asset_post_verifier,
)


def _verifier_root(tmp_path: Path) -> tuple[Path, Path, Path]:
    root = tmp_path / "infra" / "ansible"
    inventory = root / "inventory" / "hosts.yml"
    inventory.parent.mkdir(parents=True)
    inventory.write_text(
        """
all:
  children:
    ai_web:
      hosts:
        host_188:
          ansible_host: 192.168.0.188
          ansible_user: ollama
""".strip()
        + "\n",
        encoding="utf-8",
    )
    key = tmp_path / "id_ed25519"
    known_hosts = tmp_path / "known_hosts"
    key.write_text("test-key-metadata-only\n", encoding="utf-8")
    known_hosts.write_text("host-key-metadata-only\n", encoding="utf-8")
    return root, key, known_hosts


def test_asset_postcondition_registry_covers_every_ansible_catalog() -> None:
    catalog_ids = {row["catalog_id"] for row in list_ansible_catalog()}

    assert registered_catalog_ids() == catalog_ids


def test_read_only_probe_command_uses_structured_inventory(tmp_path: Path) -> None:
    root, key, known_hosts = _verifier_root(tmp_path)
    condition = AssetPostcondition(
        "momo_script",
        "service",
        "host_188",
        "test -x /home/ollama/bin/momo-pg-backup.sh",
    )

    command = build_read_only_probe_command(
        condition,
        inventory_path=root / "inventory" / "hosts.yml",
        ssh_key_path=key,
        known_hosts_path=known_hosts,
    )

    assert command[0] == "ssh"
    assert "ollama@192.168.0.188" in command
    assert "ansible-playbook" not in command
    assert command[-1] == condition.probe


@pytest.mark.asyncio
async def test_independent_post_verifier_requires_every_asset_probe(
    tmp_path: Path,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    root, key, known_hosts = _verifier_root(tmp_path)
    monkeypatch.setenv("AWOOOI_BUILD_COMMIT_SHA", "source-sha-123")
    calls = 0

    async def probe_runner(
        _command: list[str],
        *,
        timeout_seconds: int,
    ) -> ReadOnlyProbeResult:
        nonlocal calls
        assert timeout_seconds == 7
        calls += 1
        return ReadOnlyProbeResult(
            returncode=0 if calls < 5 else 1,
            stdout="runtime probe output must not be persisted",
            stderr="",
            duration_ms=2,
        )

    receipt = await run_ansible_asset_post_verifier(
        catalog_id="ansible:188-momo-backup-user",
        automation_run_id="run-1",
        apply_op_id="apply-1",
        inventory_hosts=("host_188",),
        executor_returncode=0,
        playbook_root=root,
        ssh_key_path=key,
        known_hosts_path=known_hosts,
        timeout_seconds=7,
        probe_runner=probe_runner,
    )

    assert calls == 5
    assert receipt["verification_result"] == "failed"
    assert receipt["all_postconditions_passed"] is False
    assert receipt["passed_postcondition_count"] == 4
    assert receipt["required_postcondition_count"] == 5
    assert receipt["executor_returncode_trusted"] is False
    assert receipt["raw_output_stored"] is False
    assert "postcondition_failed:host_188_momo_backup_metric" in receipt[
        "active_blockers"
    ]
    assert all("stdout" not in item and "stderr" not in item for item in receipt["postconditions"])
    assert all(len(item["output_sha256"]) == 64 for item in receipt["postconditions"])


@pytest.mark.asyncio
async def test_executor_failure_never_runs_or_passes_independent_verifier(
    tmp_path: Path,
) -> None:
    root, key, known_hosts = _verifier_root(tmp_path)
    called = False

    async def probe_runner(*_args, **_kwargs) -> ReadOnlyProbeResult:
        nonlocal called
        called = True
        return ReadOnlyProbeResult(0, "", "", 1)

    receipt = await run_ansible_asset_post_verifier(
        catalog_id="ansible:188-momo-backup-user",
        automation_run_id="run-1",
        apply_op_id="apply-1",
        inventory_hosts=("host_188",),
        executor_returncode=2,
        playbook_root=root,
        ssh_key_path=key,
        known_hosts_path=known_hosts,
        probe_runner=probe_runner,
    )

    assert called is False
    assert receipt["verification_result"] == "failed"
    assert "executor_apply_failed_before_post_verifier" in receipt[
        "active_blockers"
    ]


def test_executor_returncode_alone_cannot_claim_post_verifier_success() -> None:
    result = AnsibleRunResult(returncode=0, stdout="", stderr="", duration_ms=1)
    verified = {
        "verification_result": "success",
        "all_postconditions_passed": True,
        "executor_returncode_trusted": False,
    }

    assert _post_apply_verification_result(result) == "failed"
    assert _post_apply_verification_result(result, verified) == "success"
    assert _post_apply_verification_result(
        result,
        {**verified, "all_postconditions_passed": False},
    ) == "failed"


def test_verifier_failure_enters_durable_failed_apply_and_retry_first() -> None:
    terminal_source = inspect.getsource(_record_apply_post_verifier_terminal)
    broker_source = inspect.getsource(run_pending_check_modes_once)

    assert "ELSE 'failed'" in terminal_source
    assert "independent_post_verifier_failed" in terminal_source
    assert "run_failed_apply_check_mode_replay_once" in broker_source
    assert "controlled_apply_verifier_failed" in broker_source