from __future__ import annotations from pathlib import Path import yaml ROOT = Path(__file__).resolve().parents[3] ALERTS = ROOT / "ops" / "monitoring" / "alerts-unified.yml" LEGACY_ALERTS = ROOT / "ops" / "monitoring" / "alerts.yml" COLD_START_PLAYBOOKS = ROOT / "scripts" / "cold_start_playbooks.py" def load_alerts() -> dict[str, dict]: payload = yaml.safe_load(ALERTS.read_text(encoding="utf-8")) alerts: dict[str, dict] = {} for group in payload["groups"]: for rule in group.get("rules", []): if "alert" in rule: alerts[rule["alert"]] = rule return alerts def test_110_moderate_pressure_alert_routes_to_live_controller() -> None: alerts = load_alerts() rule = alerts["Host110SustainedModeratePressure"] expr = str(rule["expr"]) annotations = rule["annotations"] action = annotations["auto_repair_action"] assert 'awoooi_host_load5_per_core{host="110"} > 0.75' in expr assert 'docker_container_cpu_cores{host="110"' in expr assert 'awoooi_host_process_family_cpu_percent{host="110"' in expr assert "> 1.0" in expr assert "> 50" in expr assert "systemd_control_plane" in expr assert "gitea" in expr assert "stockplatform-v2-postgres-1" in expr assert rule["for"] == "1m" assert rule["labels"]["auto_repair"] == "true" assert "/home/wooo/scripts/host-sustained-load-controller.py" in action assert "--load5-per-core-threshold 0.75" in action assert "--hot-container-cpu-threshold 1.0" in action assert "--container-cpu-threshold 2.0" in action assert "--process-family-cpu-threshold 50" in action assert "不讀 secret" in annotations["runbook"] assert "禁止 Docker / systemd / Nginx / DB restart" in annotations["runbook"] def test_critical_sustained_load_alert_uses_deployed_controller_path() -> None: alerts = load_alerts() action = alerts["HostLoadAverageSustainedHigh"]["annotations"]["auto_repair_action"] assert "/home/wooo/scripts/host-sustained-load-controller.py" in action assert "scripts/ops/host-sustained-load-controller.py" not in action def test_host_memory_pressure_is_not_mislabeled_as_oom_or_host_188() -> None: alerts = load_alerts() rule = alerts["HostMemoryUsageHigh"] annotations = rule["annotations"] assert "HostOutOfMemory" not in alerts assert rule["labels"]["layer"] == "systemd-{{ $labels.host }}" assert rule["labels"]["auto_repair"] == "true" assert "不代表已發生 OOM" in annotations["description"] assert "oom_kill" in annotations["auto_repair_action"] assert "drop_caches" not in annotations["auto_repair_action"] assert "restart" not in annotations["auto_repair_action"] assert "reboot" not in annotations["auto_repair_action"] assert "禁止 drop_caches、restart、reboot" in annotations["runbook"] def test_legacy_rule_and_seed_playbook_use_same_read_only_memory_contract() -> None: legacy_payload = yaml.safe_load(LEGACY_ALERTS.read_text(encoding="utf-8")) legacy_alerts = { rule["alert"]: rule for group in legacy_payload["groups"] for rule in group.get("rules", []) if "alert" in rule } playbook_source = COLD_START_PLAYBOOKS.read_text(encoding="utf-8") assert "HostMemoryUsageHigh" in legacy_alerts assert "HostOutOfMemory" not in legacy_alerts assert legacy_alerts["HostMemoryUsageHigh"]["labels"]["layer"] == "systemd-{{ $labels.host }}" assert "sudo sysctl vm.drop_caches" not in playbook_source assert '"alert_type": "HostMemoryUsageHigh"' in playbook_source assert '"HostOutOfMemory"' in playbook_source # historical persisted-event alias only def test_backup_aggregate_alert_excludes_old_wrapper_noise() -> None: alerts = load_alerts() expr = str(alerts["BackupAggregateRunFailed"]["expr"]) assert 'awoooi_backup_last_run_failed_count{host="110",exported_job!="backup_all"}' in expr assert 'exported_job="backup_all"} > 0' not in expr def test_reboot_slo_alerts_project_named_blockers_to_telegram() -> None: alerts = load_alerts() blocker = alerts["RebootAutoRecoveryActiveBlocker"] metric_missing = alerts["RebootAutoRecoveryActiveBlockerMetricMissing"] assert "awoooi_reboot_auto_recovery_slo_active_blocker" in str(blocker["expr"]) assert blocker["labels"]["notification_type"] == "TYPE-3" assert "$labels.blocker" in blocker["annotations"]["summary"] assert "$labels.blocker" in blocker["annotations"]["description"] assert "Windows99" in blocker["annotations"]["runbook"] assert "backup-status" in blocker["annotations"]["runbook"] assert "禁止 reboot" in blocker["annotations"]["runbook"] assert "awoooi_reboot_auto_recovery_slo_blocker_count" in str(metric_missing["expr"]) assert "awoooi_reboot_auto_recovery_slo_active_blocker" in str(metric_missing["expr"]) assert metric_missing["labels"]["notification_type"] == "TYPE-1"