From 22885df82d0ba7b894591732a6780340392621a5 Mon Sep 17 00:00:00 2001 From: ogt Date: Wed, 15 Jul 2026 07:59:17 +0800 Subject: [PATCH] fix(monitoring): target host ssh endpoint --- ops/monitoring/alerts-unified.yml | 2 +- ops/monitoring/alerts.yml | 2 +- scripts/ops/tests/test_host_pressure_alert_contract.py | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/ops/monitoring/alerts-unified.yml b/ops/monitoring/alerts-unified.yml index 56e23e0e0..177759c68 100644 --- a/ops/monitoring/alerts-unified.yml +++ b/ops/monitoring/alerts-unified.yml @@ -119,7 +119,7 @@ groups: summary: "主機 {{ $labels.host }} 記憶體使用率偏高" description: "MemAvailable 推算使用率超過 85% 持續 5 分鐘；這是容量壓力訊號，不代表已發生 OOM、服務停止或主機關機。" auto_repair_action: >- - ssh {{ $labels.instance }} 'free -h; grep "^oom_kill " /proc/vmstat; ps aux --sort=-%mem | head -20' + ssh 192.168.0.{{ $labels.host }} 'free -h; grep "^oom_kill " /proc/vmstat; ps aux --sort=-%mem | head -20' runbook: "唯讀確認 MemAvailable、swap、/proc/vmstat oom_kill、主要記憶體程序與服務健康；禁止 drop_caches、restart、reboot。若 oom_kill 未增加且服務健康，維持 degraded/capacity-pressure，不得誤報 OOM 或主機離線。" - alert: HostOutOfDiskSpace diff --git a/ops/monitoring/alerts.yml b/ops/monitoring/alerts.yml index c89d57a78..2897126a3 100644 --- a/ops/monitoring/alerts.yml +++ b/ops/monitoring/alerts.yml @@ -87,7 +87,7 @@ groups: summary: "主機 {{ $labels.host }} 記憶體使用率偏高" description: "MemAvailable 推算使用率超過 85% 持續 5 分鐘；這是容量壓力訊號，不代表已發生 OOM、服務停止或主機關機。" auto_repair_action: >- - ssh {{ $labels.instance }} 'free -h; grep "^oom_kill " /proc/vmstat; ps aux --sort=-%mem | head -20' + ssh 192.168.0.{{ $labels.host }} 'free -h; grep "^oom_kill " /proc/vmstat; ps aux --sort=-%mem | head -20' runbook: "唯讀確認 MemAvailable、swap、/proc/vmstat oom_kill、主要記憶體程序與服務健康；禁止 drop_caches、restart、reboot。若 oom_kill 未增加且服務健康，維持 degraded/capacity-pressure，不得誤報 OOM 或主機離線。" - alert: HostOutOfDiskSpace diff --git a/scripts/ops/tests/test_host_pressure_alert_contract.py b/scripts/ops/tests/test_host_pressure_alert_contract.py index 6cd99f203..88f634205 100644 --- a/scripts/ops/tests/test_host_pressure_alert_contract.py +++ b/scripts/ops/tests/test_host_pressure_alert_contract.py @@ -65,6 +65,8 @@ def test_host_memory_pressure_is_not_mislabeled_as_oom_or_host_188() -> None: assert rule["labels"]["layer"] == "systemd-{{ $labels.host }}" assert rule["labels"]["auto_repair"] == "true" assert "不代表已發生 OOM" in annotations["description"] + assert annotations["auto_repair_action"].startswith("ssh 192.168.0.{{ $labels.host }} ") + assert "{{ $labels.instance }}" not in annotations["auto_repair_action"] assert "oom_kill" in annotations["auto_repair_action"] assert "drop_caches" not in annotations["auto_repair_action"] assert "restart" not in annotations["auto_repair_action"]