fix(monitoring): classify host memory pressure accurately

This commit is contained in:
ogt
2026-07-15 06:15:13 +08:00
parent 89595a7d8d
commit 5c0c5f6da2
11 changed files with 76 additions and 25 deletions

View File

@@ -93,7 +93,7 @@ inhibit_rules:
- source_matchers:
- alertname="HostDown"
target_matchers:
- alertname=~"HostHighCpuLoad|HostOutOfMemory|HostOutOfDiskSpace"
- alertname=~"HostHighCpuLoad|HostMemoryUsageHigh|HostOutOfMemory|HostOutOfDiskSpace"
equal: ['host']
- source_matchers:

View File

@@ -104,20 +104,23 @@ groups:
auto_repair_action: "ssh 192.168.0.{{ $labels.host }} '/home/wooo/scripts/host-sustained-load-controller.py --host {{ $labels.host }} --metrics-file /home/wooo/node_exporter_textfiles/host_runaway_process.prom --docker-stats-file /home/wooo/node_exporter_textfiles/docker_stats.prom --json'"
runbook: "交給 host-sustained-load-controller 產生 AI controlled packetorphan browser 走 host-runaway-process-remediation.py dry-run → controlled SIGTERM → verifier合法 CI/BuildKit 走 runner pressure fail-closed 與 drain/cancel packetunknown 先跑 host-sustained-load-evidence.py 只讀脫敏證據再選服務專屬 PlayBookswap 走服務專屬記憶體 PlayBook。禁止直接 docker/systemd/nginx/firewall/reboot。"
- alert: HostOutOfMemory
- alert: HostMemoryUsageHigh
expr: (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100 > 85
for: 5m
labels:
severity: warning
layer: systemd-188
layer: 'systemd-{{ $labels.host }}'
team: ops
auto_repair: "true"
mcp_provider: "ssh_host"
host_type: "bare_metal"
alert_category: "host_resource"
annotations:
summary: "主機 {{ $labels.host }} 記憶體不足"
description: "記憶體使用率超過 85%"
summary: "主機 {{ $labels.host }} 記憶體使用率偏高"
description: "MemAvailable 推算使用率超過 85% 持續 5 分鐘;這是容量壓力訊號,不代表已發生 OOM、服務停止或主機關機。"
auto_repair_action: >-
ssh {{ $labels.instance }} 'free -h; grep "^oom_kill " /proc/vmstat; ps aux --sort=-%mem | head -20'
runbook: "唯讀確認 MemAvailable、swap、/proc/vmstat oom_kill、主要記憶體程序與服務健康禁止 drop_caches、restart、reboot。若 oom_kill 未增加且服務健康,維持 degraded/capacity-pressure不得誤報 OOM 或主機離線。"
- alert: HostOutOfDiskSpace
expr: (1 - (node_filesystem_avail_bytes{fstype!~"tmpfs|overlay"} / node_filesystem_size_bytes{fstype!~"tmpfs|overlay"})) * 100 > 85

View File

@@ -72,23 +72,23 @@ groups:
auto_repair_action: "ssh 192.168.0.{{ $labels.host }} '/home/wooo/scripts/host-sustained-load-controller.py --host {{ $labels.host }} --metrics-file /home/wooo/node_exporter_textfiles/host_runaway_process.prom --docker-stats-file /home/wooo/node_exporter_textfiles/docker_stats.prom --json'"
runbook: "交給 host-sustained-load-controller 產生 AI controlled packetorphan browser 走 host-runaway-process-remediation.py dry-run → controlled SIGTERM → verifier合法 CI/BuildKit 走 runner pressure fail-closed 與 drain/cancel packetunknown 先跑 host-sustained-load-evidence.py 只讀脫敏證據再選服務專屬 PlayBookswap 走服務專屬記憶體 PlayBook。禁止直接 docker/systemd/nginx/firewall/reboot。"
- alert: HostOutOfMemory
- alert: HostMemoryUsageHigh
expr: (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100 > 85
for: 5m
labels:
severity: warning
layer: systemd-188
layer: 'systemd-{{ $labels.host }}'
team: ops
auto_repair: "true"
mcp_provider: "ssh_host"
host_type: "bare_metal"
alert_category: "host_resource"
annotations:
summary: "主機 {{ $labels.host }} 記憶體不足"
description: "記憶體使用率超過 85%"
# 2026-05-02 ogt + Claude Sonnet 4.6: 引導 LLM 走 SSH 診斷
auto_repair_action: "ssh {{ $labels.instance }} 'ps aux --sort=-%mem | head -20' (host 記憶體診斷;禁 kubectl restart — 主因常為第三方服務)"
runbook: "host 記憶體不足排查SSH 看 top 進程;若為第三方服務需擴容或調 limit"
summary: "主機 {{ $labels.host }} 記憶體使用率偏高"
description: "MemAvailable 推算使用率超過 85% 持續 5 分鐘;這是容量壓力訊號,不代表已發生 OOM、服務停止或主機關機。"
auto_repair_action: >-
ssh {{ $labels.instance }} 'free -h; grep "^oom_kill " /proc/vmstat; ps aux --sort=-%mem | head -20'
runbook: "唯讀確認 MemAvailable、swap、/proc/vmstat oom_kill、主要記憶體程序與服務健康禁止 drop_caches、restart、reboot。若 oom_kill 未增加且服務健康,維持 degraded/capacity-pressure不得誤報 OOM 或主機離線。"
- alert: HostOutOfDiskSpace
expr: (1 - (node_filesystem_avail_bytes{fstype!~"tmpfs|overlay"} / node_filesystem_size_bytes{fstype!~"tmpfs|overlay"})) * 100 > 85