diff --git a/ops/monitoring/alerts-unified.yml b/ops/monitoring/alerts-unified.yml index 177759c68..a99a7c197 100644 --- a/ops/monitoring/alerts-unified.yml +++ b/ops/monitoring/alerts-unified.yml @@ -119,7 +119,7 @@ groups: summary: "主機 {{ $labels.host }} 記憶體使用率偏高" description: "MemAvailable 推算使用率超過 85% 持續 5 分鐘;這是容量壓力訊號,不代表已發生 OOM、服務停止或主機關機。" auto_repair_action: >- - ssh 192.168.0.{{ $labels.host }} 'free -h; grep "^oom_kill " /proc/vmstat; ps aux --sort=-%mem | head -20' + ssh {{ if eq $labels.host "112" }}kali@{{ end }}192.168.0.{{ $labels.host }} 'free -h; grep "^oom_kill " /proc/vmstat; ps aux --sort=-%mem | head -20' runbook: "唯讀確認 MemAvailable、swap、/proc/vmstat oom_kill、主要記憶體程序與服務健康;禁止 drop_caches、restart、reboot。若 oom_kill 未增加且服務健康,維持 degraded/capacity-pressure,不得誤報 OOM 或主機離線。" - alert: HostOutOfDiskSpace diff --git a/ops/monitoring/alerts.yml b/ops/monitoring/alerts.yml index 2897126a3..9b8fceb5b 100644 --- a/ops/monitoring/alerts.yml +++ b/ops/monitoring/alerts.yml @@ -87,7 +87,7 @@ groups: summary: "主機 {{ $labels.host }} 記憶體使用率偏高" description: "MemAvailable 推算使用率超過 85% 持續 5 分鐘;這是容量壓力訊號,不代表已發生 OOM、服務停止或主機關機。" auto_repair_action: >- - ssh 192.168.0.{{ $labels.host }} 'free -h; grep "^oom_kill " /proc/vmstat; ps aux --sort=-%mem | head -20' + ssh {{ if eq $labels.host "112" }}kali@{{ end }}192.168.0.{{ $labels.host }} 'free -h; grep "^oom_kill " /proc/vmstat; ps aux --sort=-%mem | head -20' runbook: "唯讀確認 MemAvailable、swap、/proc/vmstat oom_kill、主要記憶體程序與服務健康;禁止 drop_caches、restart、reboot。若 oom_kill 未增加且服務健康,維持 degraded/capacity-pressure,不得誤報 OOM 或主機離線。" - alert: HostOutOfDiskSpace diff --git a/scripts/ops/tests/test_host_pressure_alert_contract.py b/scripts/ops/tests/test_host_pressure_alert_contract.py index 88f634205..92808411d 100644 --- a/scripts/ops/tests/test_host_pressure_alert_contract.py +++ b/scripts/ops/tests/test_host_pressure_alert_contract.py @@ -65,7 +65,9 @@ def test_host_memory_pressure_is_not_mislabeled_as_oom_or_host_188() -> None: assert rule["labels"]["layer"] == "systemd-{{ $labels.host }}" assert rule["labels"]["auto_repair"] == "true" assert "不代表已發生 OOM" in annotations["description"] - assert annotations["auto_repair_action"].startswith("ssh 192.168.0.{{ $labels.host }} ") + assert annotations["auto_repair_action"].startswith( + 'ssh {{ if eq $labels.host "112" }}kali@{{ end }}192.168.0.{{ $labels.host }} ' + ) assert "{{ $labels.instance }}" not in annotations["auto_repair_action"] assert "oom_kill" in annotations["auto_repair_action"] assert "drop_caches" not in annotations["auto_repair_action"]