fix(monitoring): classify host memory pressure accurately
This commit is contained in:
@@ -93,7 +93,7 @@ inhibit_rules:
|
||||
- source_matchers:
|
||||
- alertname="HostDown"
|
||||
target_matchers:
|
||||
- alertname=~"HostHighCpuLoad|HostOutOfMemory|HostOutOfDiskSpace"
|
||||
- alertname=~"HostHighCpuLoad|HostMemoryUsageHigh|HostOutOfMemory|HostOutOfDiskSpace"
|
||||
equal: ['host']
|
||||
|
||||
- source_matchers:
|
||||
|
||||
@@ -104,20 +104,23 @@ groups:
|
||||
auto_repair_action: "ssh 192.168.0.{{ $labels.host }} '/home/wooo/scripts/host-sustained-load-controller.py --host {{ $labels.host }} --metrics-file /home/wooo/node_exporter_textfiles/host_runaway_process.prom --docker-stats-file /home/wooo/node_exporter_textfiles/docker_stats.prom --json'"
|
||||
runbook: "交給 host-sustained-load-controller 產生 AI controlled packet:orphan browser 走 host-runaway-process-remediation.py dry-run → controlled SIGTERM → verifier;合法 CI/BuildKit 走 runner pressure fail-closed 與 drain/cancel packet;unknown 先跑 host-sustained-load-evidence.py 只讀脫敏證據再選服務專屬 PlayBook;swap 走服務專屬記憶體 PlayBook。禁止直接 docker/systemd/nginx/firewall/reboot。"
|
||||
|
||||
- alert: HostOutOfMemory
|
||||
- alert: HostMemoryUsageHigh
|
||||
expr: (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100 > 85
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
layer: systemd-188
|
||||
layer: 'systemd-{{ $labels.host }}'
|
||||
team: ops
|
||||
auto_repair: "true"
|
||||
mcp_provider: "ssh_host"
|
||||
host_type: "bare_metal"
|
||||
alert_category: "host_resource"
|
||||
annotations:
|
||||
summary: "主機 {{ $labels.host }} 記憶體不足"
|
||||
description: "記憶體使用率超過 85%"
|
||||
summary: "主機 {{ $labels.host }} 記憶體使用率偏高"
|
||||
description: "MemAvailable 推算使用率超過 85% 持續 5 分鐘;這是容量壓力訊號,不代表已發生 OOM、服務停止或主機關機。"
|
||||
auto_repair_action: >-
|
||||
ssh {{ $labels.instance }} 'free -h; grep "^oom_kill " /proc/vmstat; ps aux --sort=-%mem | head -20'
|
||||
runbook: "唯讀確認 MemAvailable、swap、/proc/vmstat oom_kill、主要記憶體程序與服務健康;禁止 drop_caches、restart、reboot。若 oom_kill 未增加且服務健康,維持 degraded/capacity-pressure,不得誤報 OOM 或主機離線。"
|
||||
|
||||
- alert: HostOutOfDiskSpace
|
||||
expr: (1 - (node_filesystem_avail_bytes{fstype!~"tmpfs|overlay"} / node_filesystem_size_bytes{fstype!~"tmpfs|overlay"})) * 100 > 85
|
||||
|
||||
@@ -72,23 +72,23 @@ groups:
|
||||
auto_repair_action: "ssh 192.168.0.{{ $labels.host }} '/home/wooo/scripts/host-sustained-load-controller.py --host {{ $labels.host }} --metrics-file /home/wooo/node_exporter_textfiles/host_runaway_process.prom --docker-stats-file /home/wooo/node_exporter_textfiles/docker_stats.prom --json'"
|
||||
runbook: "交給 host-sustained-load-controller 產生 AI controlled packet:orphan browser 走 host-runaway-process-remediation.py dry-run → controlled SIGTERM → verifier;合法 CI/BuildKit 走 runner pressure fail-closed 與 drain/cancel packet;unknown 先跑 host-sustained-load-evidence.py 只讀脫敏證據再選服務專屬 PlayBook;swap 走服務專屬記憶體 PlayBook。禁止直接 docker/systemd/nginx/firewall/reboot。"
|
||||
|
||||
- alert: HostOutOfMemory
|
||||
- alert: HostMemoryUsageHigh
|
||||
expr: (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100 > 85
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
layer: systemd-188
|
||||
layer: 'systemd-{{ $labels.host }}'
|
||||
team: ops
|
||||
auto_repair: "true"
|
||||
mcp_provider: "ssh_host"
|
||||
host_type: "bare_metal"
|
||||
alert_category: "host_resource"
|
||||
annotations:
|
||||
summary: "主機 {{ $labels.host }} 記憶體不足"
|
||||
description: "記憶體使用率超過 85%"
|
||||
# 2026-05-02 ogt + Claude Sonnet 4.6: 引導 LLM 走 SSH 診斷
|
||||
auto_repair_action: "ssh {{ $labels.instance }} 'ps aux --sort=-%mem | head -20' (host 記憶體診斷;禁 kubectl restart — 主因常為第三方服務)"
|
||||
runbook: "host 記憶體不足排查:SSH 看 top 進程;若為第三方服務需擴容或調 limit"
|
||||
summary: "主機 {{ $labels.host }} 記憶體使用率偏高"
|
||||
description: "MemAvailable 推算使用率超過 85% 持續 5 分鐘;這是容量壓力訊號,不代表已發生 OOM、服務停止或主機關機。"
|
||||
auto_repair_action: >-
|
||||
ssh {{ $labels.instance }} 'free -h; grep "^oom_kill " /proc/vmstat; ps aux --sort=-%mem | head -20'
|
||||
runbook: "唯讀確認 MemAvailable、swap、/proc/vmstat oom_kill、主要記憶體程序與服務健康;禁止 drop_caches、restart、reboot。若 oom_kill 未增加且服務健康,維持 degraded/capacity-pressure,不得誤報 OOM 或主機離線。"
|
||||
|
||||
- alert: HostOutOfDiskSpace
|
||||
expr: (1 - (node_filesystem_avail_bytes{fstype!~"tmpfs|overlay"} / node_filesystem_size_bytes{fstype!~"tmpfs|overlay"})) * 100 > 85
|
||||
|
||||
Reference in New Issue
Block a user