awoooi/scripts/ops/tests/test_host_pressure_alert_contract.py

from __future__ import annotations

from pathlib import Path

import yaml


ROOT = Path(__file__).resolve().parents[3]
ALERTS = ROOT / "ops" / "monitoring" / "alerts-unified.yml"
LEGACY_ALERTS = ROOT / "ops" / "monitoring" / "alerts.yml"
COLD_START_PLAYBOOKS = ROOT / "scripts" / "cold_start_playbooks.py"


def load_alerts() -> dict[str, dict]:
    payload = yaml.safe_load(ALERTS.read_text(encoding="utf-8"))
    alerts: dict[str, dict] = {}
    for group in payload["groups"]:
        for rule in group.get("rules", []):
            if "alert" in rule:
                alerts[rule["alert"]] = rule
    return alerts


def test_110_moderate_pressure_alert_routes_to_live_controller() -> None:
    alerts = load_alerts()
    rule = alerts["Host110SustainedModeratePressure"]

    expr = str(rule["expr"])
    annotations = rule["annotations"]
    action = annotations["auto_repair_action"]

    assert 'awoooi_host_load5_per_core{host="110"} > 0.75' in expr
    assert 'docker_container_cpu_cores{host="110"' in expr
    assert 'awoooi_host_process_family_cpu_percent{host="110"' in expr
    assert "> 1.0" in expr
    assert "> 50" in expr
    assert "systemd_control_plane" in expr
    assert "gitea" in expr
    assert "stockplatform-v2-postgres-1" in expr
    assert rule["for"] == "1m"
    assert rule["labels"]["auto_repair"] == "true"
    assert "/home/wooo/scripts/host-sustained-load-controller.py" in action
    assert "--load5-per-core-threshold 0.75" in action
    assert "--hot-container-cpu-threshold 1.0" in action
    assert "--container-cpu-threshold 2.0" in action
    assert "--process-family-cpu-threshold 50" in action
    assert "不讀 secret" in annotations["runbook"]
    assert "禁止 Docker / systemd / Nginx / DB restart" in annotations["runbook"]


def test_critical_sustained_load_alert_uses_deployed_controller_path() -> None:
    alerts = load_alerts()
    action = alerts["HostLoadAverageSustainedHigh"]["annotations"]["auto_repair_action"]

    assert "/home/wooo/scripts/host-sustained-load-controller.py" in action
    assert "scripts/ops/host-sustained-load-controller.py" not in action


def test_host_memory_pressure_is_not_mislabeled_as_oom_or_host_188() -> None:
    alerts = load_alerts()
    rule = alerts["HostMemoryUsageHigh"]
    annotations = rule["annotations"]

    assert "HostOutOfMemory" not in alerts
    assert rule["labels"]["layer"] == "systemd-{{ $labels.host }}"
    assert rule["labels"]["auto_repair"] == "true"
    assert "不代表已發生 OOM" in annotations["description"]
    assert annotations["auto_repair_action"].startswith(
        'ssh {{ if eq $labels.host "112" }}kali@{{ end }}192.168.0.{{ $labels.host }} '
    )
    assert "{{ $labels.instance }}" not in annotations["auto_repair_action"]
    assert "oom_kill" in annotations["auto_repair_action"]
    assert "drop_caches" not in annotations["auto_repair_action"]
    assert "restart" not in annotations["auto_repair_action"]
    assert "reboot" not in annotations["auto_repair_action"]
    assert "禁止 drop_caches、restart、reboot" in annotations["runbook"]


def test_legacy_rule_and_seed_playbook_use_same_read_only_memory_contract() -> None:
    legacy_payload = yaml.safe_load(LEGACY_ALERTS.read_text(encoding="utf-8"))
    legacy_alerts = {
        rule["alert"]: rule
        for group in legacy_payload["groups"]
        for rule in group.get("rules", [])
        if "alert" in rule
    }
    playbook_source = COLD_START_PLAYBOOKS.read_text(encoding="utf-8")

    assert "HostMemoryUsageHigh" in legacy_alerts
    assert "HostOutOfMemory" not in legacy_alerts
    assert legacy_alerts["HostMemoryUsageHigh"]["labels"]["layer"] == "systemd-{{ $labels.host }}"
    assert "sudo sysctl vm.drop_caches" not in playbook_source
    assert '"alert_type": "HostMemoryUsageHigh"' in playbook_source
    assert '"HostOutOfMemory"' in playbook_source  # historical persisted-event alias only


def test_backup_aggregate_alert_excludes_old_wrapper_noise() -> None:
    alerts = load_alerts()
    expr = str(alerts["BackupAggregateRunFailed"]["expr"])

    assert 'awoooi_backup_last_run_failed_count{host="110",exported_job!="backup_all"}' in expr
    assert 'exported_job="backup_all"} > 0' not in expr


def test_reboot_slo_alerts_project_named_blockers_to_telegram() -> None:
    alerts = load_alerts()
    blocker = alerts["RebootAutoRecoveryActiveBlocker"]
    metric_missing = alerts["RebootAutoRecoveryActiveBlockerMetricMissing"]

    assert "awoooi_reboot_auto_recovery_slo_active_blocker" in str(blocker["expr"])
    assert blocker["labels"]["notification_type"] == "TYPE-3"
    assert "$labels.blocker" in blocker["annotations"]["summary"]
    assert "$labels.blocker" in blocker["annotations"]["description"]
    assert "Windows99" in blocker["annotations"]["runbook"]
    assert "backup-status" in blocker["annotations"]["runbook"]
    assert "禁止 reboot" in blocker["annotations"]["runbook"]

    assert "awoooi_reboot_auto_recovery_slo_blocker_count" in str(metric_missing["expr"])
    assert "awoooi_reboot_auto_recovery_slo_active_blocker" in str(metric_missing["expr"])
    assert metric_missing["labels"]["notification_type"] == "TYPE-1"