From 5c0c5f6da2e373a37457a94de8966c8c8252286b Mon Sep 17 00:00:00 2001 From: ogt Date: Wed, 15 Jul 2026 06:15:13 +0800 Subject: [PATCH] fix(monitoring): classify host memory pressure accurately --- apps/api/src/constants/alert_types.py | 4 ++- .../ai_agent_autonomous_runtime_control.py | 2 +- apps/api/src/services/decision_manager.py | 3 +- apps/api/tests/test_classify_alert_early.py | 4 +++ apps/api/tests/test_get_incident_type.py | 5 +++ apps/sensor/agent.py | 4 +-- ops/alertmanager/alertmanager.yml | 2 +- ops/monitoring/alerts-unified.yml | 11 +++--- ops/monitoring/alerts.yml | 14 ++++---- scripts/cold_start_playbooks.py | 16 ++++----- .../test_host_pressure_alert_contract.py | 36 +++++++++++++++++++ 11 files changed, 76 insertions(+), 25 deletions(-) diff --git a/apps/api/src/constants/alert_types.py b/apps/api/src/constants/alert_types.py index 48acfd012..590585cba 100644 --- a/apps/api/src/constants/alert_types.py +++ b/apps/api/src/constants/alert_types.py @@ -11,11 +11,13 @@ alertname → incident_type 靜態對應表 擴展方式：在 alert_rules.yaml 中新增 incident_type 欄位；此 dict 僅需補無 YAML 規則的 alertname。 """ -# alertname → incident_type 對應（56 筆） +# alertname → incident_type 對應；由測試驗證必要 taxonomy，不用易漂移的手填筆數。 ALERTNAME_TO_TYPE: dict[str, str] = { # --- 主機層 (host_alerts) --- "HostDown": "host_down", "HostHighCpuLoad": "host_cpu", + "HostMemoryUsageHigh": "host_memory", + # Historical alias for persisted incidents created before the taxonomy fix. "HostOutOfMemory": "host_memory", "HostOutOfDiskSpace": "disk_full", "HostBackupFailed": "backup_failure", diff --git a/apps/api/src/services/ai_agent_autonomous_runtime_control.py b/apps/api/src/services/ai_agent_autonomous_runtime_control.py index 99e39a130..294f3afe2 100644 --- a/apps/api/src/services/ai_agent_autonomous_runtime_control.py +++ b/apps/api/src/services/ai_agent_autonomous_runtime_control.py @@ -2661,7 +2661,7 @@ def _build_host_sustained_load_controlled_automation_readback() -> dict[str, Any }, { "class_id": "memory_or_swap_pressure", - "alertnames": ["HostLoadAverageSustainedHigh", "HostOutOfMemory"], + "alertnames": ["HostLoadAverageSustainedHigh", "HostMemoryUsageHigh", "HostOutOfMemory"], "classifier": "host-sustained-load-controller.py:blocked_memory_or_swap_pressure_requires_service_playbook", "controlled_action": "route to service-specific memory/cgroup playbook with check-mode diff", "controlled_apply_allowed": False, diff --git a/apps/api/src/services/decision_manager.py b/apps/api/src/services/decision_manager.py index afbe5964f..2259e6129 100644 --- a/apps/api/src/services/decision_manager.py +++ b/apps/api/src/services/decision_manager.py @@ -960,6 +960,7 @@ async def _resolve_target_from_k8s(incident: "Incident", namespace: str) -> str # 並加 log 便於追蹤 fallback 路徑 _ALERTNAME_KEYWORDS: dict[str, list[str]] = { "HostHighCpuLoad": ["api", "web"], + "HostMemoryUsageHigh": ["api", "web"], "HostOutOfMemory": ["api", "web"], "DockerContainerUnhealthy": [], "DockerContainerExited": [], @@ -1064,7 +1065,7 @@ async def _fetch_metrics_snapshot(incident: Incident) -> dict: snapshots: dict = {} # 根據 alertname 選擇最相關的指標 - if alertname in ("HostHighCpuLoad", "HostOutOfMemory"): + if alertname in ("HostHighCpuLoad", "HostMemoryUsageHigh", "HostOutOfMemory"): if instance: host = instance.split(":")[0] # P0 fix 2026-04-11: _instant_query 要求 dict，回傳 {"result": [...]} diff --git a/apps/api/tests/test_classify_alert_early.py b/apps/api/tests/test_classify_alert_early.py index c4f133d2d..970de6c66 100644 --- a/apps/api/tests/test_classify_alert_early.py +++ b/apps/api/tests/test_classify_alert_early.py @@ -103,6 +103,10 @@ class TestInfrastructure: assert ac == "host_resource" def test_host_memory(self): + ac, nt = classify_alert_early("HostMemoryUsageHigh", "warning", {}) + assert ac == "host_resource" + + def test_historical_host_out_of_memory_alias(self): ac, nt = classify_alert_early("HostOutOfMemory", "warning", {}) assert ac == "host_resource" diff --git a/apps/api/tests/test_get_incident_type.py b/apps/api/tests/test_get_incident_type.py index e6343693b..a54983e8f 100644 --- a/apps/api/tests/test_get_incident_type.py +++ b/apps/api/tests/test_get_incident_type.py @@ -21,6 +21,11 @@ class TestGetIncidentTypeStaticFallback: result = get_incident_type("HostHighCpuLoad") assert result == "host_cpu" + @pytest.mark.parametrize("alertname", ["HostMemoryUsageHigh", "HostOutOfMemory"]) + def test_memory_pressure_taxonomy_and_historical_alias(self, alertname): + """新記憶體壓力名稱與歷史事件 alias 都維持 host_memory。""" + assert get_incident_type(alertname) == "host_memory" + def test_known_alertname_k8s(self): """KubePodCrashLooping → k8s_pod_crash""" result = get_incident_type("KubePodCrashLooping") diff --git a/apps/sensor/agent.py b/apps/sensor/agent.py index a873d31dc..cb4bd5d2d 100644 --- a/apps/sensor/agent.py +++ b/apps/sensor/agent.py @@ -150,8 +150,8 @@ def collect_node_metrics(url: str, hostname: str) -> list[dict]: mem_pct = (1.0 - mem_avail / mem_total) * 100 if mem_pct > THRESHOLDS["mem_pct_high"]: alerts.append({ - "alert_name": "HostOutOfMemory", - "severity": "critical", + "alert_name": "HostMemoryUsageHigh", + "severity": "warning", "source": "node-exporter", "namespace": "infra", "target": hostname, diff --git a/ops/alertmanager/alertmanager.yml b/ops/alertmanager/alertmanager.yml index 760140cb2..44fcbe0e1 100644 --- a/ops/alertmanager/alertmanager.yml +++ b/ops/alertmanager/alertmanager.yml @@ -93,7 +93,7 @@ inhibit_rules: - source_matchers: - alertname="HostDown" target_matchers: - - alertname=~"HostHighCpuLoad|HostOutOfMemory|HostOutOfDiskSpace" + - alertname=~"HostHighCpuLoad|HostMemoryUsageHigh|HostOutOfMemory|HostOutOfDiskSpace" equal: ['host'] - source_matchers: diff --git a/ops/monitoring/alerts-unified.yml b/ops/monitoring/alerts-unified.yml index f30809f58..56e23e0e0 100644 --- a/ops/monitoring/alerts-unified.yml +++ b/ops/monitoring/alerts-unified.yml @@ -104,20 +104,23 @@ groups: auto_repair_action: "ssh 192.168.0.{{ $labels.host }} '/home/wooo/scripts/host-sustained-load-controller.py --host {{ $labels.host }} --metrics-file /home/wooo/node_exporter_textfiles/host_runaway_process.prom --docker-stats-file /home/wooo/node_exporter_textfiles/docker_stats.prom --json'" runbook: "交給 host-sustained-load-controller 產生 AI controlled packet：orphan browser 走 host-runaway-process-remediation.py dry-run → controlled SIGTERM → verifier；合法 CI/BuildKit 走 runner pressure fail-closed 與 drain/cancel packet；unknown 先跑 host-sustained-load-evidence.py 只讀脫敏證據再選服務專屬 PlayBook；swap 走服務專屬記憶體 PlayBook。禁止直接 docker/systemd/nginx/firewall/reboot。" - - alert: HostOutOfMemory + - alert: HostMemoryUsageHigh expr: (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100 > 85 for: 5m labels: severity: warning - layer: systemd-188 + layer: 'systemd-{{ $labels.host }}' team: ops auto_repair: "true" mcp_provider: "ssh_host" host_type: "bare_metal" alert_category: "host_resource" annotations: - summary: "主機 {{ $labels.host }} 記憶體不足" - description: "記憶體使用率超過 85%" + summary: "主機 {{ $labels.host }} 記憶體使用率偏高" + description: "MemAvailable 推算使用率超過 85% 持續 5 分鐘；這是容量壓力訊號，不代表已發生 OOM、服務停止或主機關機。" + auto_repair_action: >- + ssh {{ $labels.instance }} 'free -h; grep "^oom_kill " /proc/vmstat; ps aux --sort=-%mem | head -20' + runbook: "唯讀確認 MemAvailable、swap、/proc/vmstat oom_kill、主要記憶體程序與服務健康；禁止 drop_caches、restart、reboot。若 oom_kill 未增加且服務健康，維持 degraded/capacity-pressure，不得誤報 OOM 或主機離線。" - alert: HostOutOfDiskSpace expr: (1 - (node_filesystem_avail_bytes{fstype!~"tmpfs|overlay"} / node_filesystem_size_bytes{fstype!~"tmpfs|overlay"})) * 100 > 85 diff --git a/ops/monitoring/alerts.yml b/ops/monitoring/alerts.yml index 3e21b64f2..c89d57a78 100644 --- a/ops/monitoring/alerts.yml +++ b/ops/monitoring/alerts.yml @@ -72,23 +72,23 @@ groups: auto_repair_action: "ssh 192.168.0.{{ $labels.host }} '/home/wooo/scripts/host-sustained-load-controller.py --host {{ $labels.host }} --metrics-file /home/wooo/node_exporter_textfiles/host_runaway_process.prom --docker-stats-file /home/wooo/node_exporter_textfiles/docker_stats.prom --json'" runbook: "交給 host-sustained-load-controller 產生 AI controlled packet：orphan browser 走 host-runaway-process-remediation.py dry-run → controlled SIGTERM → verifier；合法 CI/BuildKit 走 runner pressure fail-closed 與 drain/cancel packet；unknown 先跑 host-sustained-load-evidence.py 只讀脫敏證據再選服務專屬 PlayBook；swap 走服務專屬記憶體 PlayBook。禁止直接 docker/systemd/nginx/firewall/reboot。" - - alert: HostOutOfMemory + - alert: HostMemoryUsageHigh expr: (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100 > 85 for: 5m labels: severity: warning - layer: systemd-188 + layer: 'systemd-{{ $labels.host }}' team: ops auto_repair: "true" mcp_provider: "ssh_host" host_type: "bare_metal" alert_category: "host_resource" annotations: - summary: "主機 {{ $labels.host }} 記憶體不足" - description: "記憶體使用率超過 85%" - # 2026-05-02 ogt + Claude Sonnet 4.6: 引導 LLM 走 SSH 診斷 - auto_repair_action: "ssh {{ $labels.instance }} 'ps aux --sort=-%mem | head -20' (host 記憶體診斷；禁 kubectl restart — 主因常為第三方服務)" - runbook: "host 記憶體不足排查：SSH 看 top 進程；若為第三方服務需擴容或調 limit" + summary: "主機 {{ $labels.host }} 記憶體使用率偏高" + description: "MemAvailable 推算使用率超過 85% 持續 5 分鐘；這是容量壓力訊號，不代表已發生 OOM、服務停止或主機關機。" + auto_repair_action: >- + ssh {{ $labels.instance }} 'free -h; grep "^oom_kill " /proc/vmstat; ps aux --sort=-%mem | head -20' + runbook: "唯讀確認 MemAvailable、swap、/proc/vmstat oom_kill、主要記憶體程序與服務健康；禁止 drop_caches、restart、reboot。若 oom_kill 未增加且服務健康，維持 degraded/capacity-pressure，不得誤報 OOM 或主機離線。" - alert: HostOutOfDiskSpace expr: (1 - (node_filesystem_avail_bytes{fstype!~"tmpfs|overlay"} / node_filesystem_size_bytes{fstype!~"tmpfs|overlay"})) * 100 > 85 diff --git a/scripts/cold_start_playbooks.py b/scripts/cold_start_playbooks.py index ce89e2d47..48708d695 100644 --- a/scripts/cold_start_playbooks.py +++ b/scripts/cold_start_playbooks.py @@ -104,18 +104,18 @@ PLAYBOOK_TEMPLATES = [ "severity_range": ["P2", "P3"], }, { - "name": "節點記憶體不足 — 清理 Cache", - "alert_type": "HostOutOfMemory", - "description": "主機記憶體不足，清理 Page Cache 釋放空間", + "name": "節點記憶體壓力 — 唯讀診斷", + "alert_type": "HostMemoryUsageHigh", + "description": "主機 MemAvailable 偏低；先驗證 OOM、swap、程序與服務狀態，不做破壞性快取清理", "category": "infrastructure", "repair_steps": [ - {"command": "ssh {target} 'free -h'", "purpose": "確認記憶體狀況"}, - {"command": "ssh {target} 'sudo sync && sudo sysctl vm.drop_caches=3'", "purpose": "清理 Page Cache"}, - {"command": "ssh {target} 'free -h'", "purpose": "確認記憶體已釋放"}, + {"command": "ssh {target} 'free -h'", "purpose": "讀取 MemAvailable 與 swap 狀況"}, + {"command": "ssh {target} \"awk '$1 == \\\"oom_kill\\\" { print }' /proc/vmstat\"", "purpose": "確認 kernel OOM kill 累計值"}, + {"command": "ssh {target} 'ps aux --sort=-%mem | head -20'", "purpose": "讀取主要記憶體程序"}, ], "estimated_minutes": 5, - "symptom_alertnames": ["HostOutOfMemory", "NodeMemoryPressure"], - "severity_range": ["P1", "P2"], + "symptom_alertnames": ["HostMemoryUsageHigh", "HostOutOfMemory", "NodeMemoryPressure"], + "severity_range": ["P2", "P3"], }, { "name": "磁碟使用率過高 — 清理舊日誌", diff --git a/scripts/ops/tests/test_host_pressure_alert_contract.py b/scripts/ops/tests/test_host_pressure_alert_contract.py index d18282895..6cd99f203 100644 --- a/scripts/ops/tests/test_host_pressure_alert_contract.py +++ b/scripts/ops/tests/test_host_pressure_alert_contract.py @@ -7,6 +7,8 @@ import yaml ROOT = Path(__file__).resolve().parents[3] ALERTS = ROOT / "ops" / "monitoring" / "alerts-unified.yml" +LEGACY_ALERTS = ROOT / "ops" / "monitoring" / "alerts.yml" +COLD_START_PLAYBOOKS = ROOT / "scripts" / "cold_start_playbooks.py" def load_alerts() -> dict[str, dict]: @@ -54,6 +56,40 @@ def test_critical_sustained_load_alert_uses_deployed_controller_path() -> None: assert "scripts/ops/host-sustained-load-controller.py" not in action +def test_host_memory_pressure_is_not_mislabeled_as_oom_or_host_188() -> None: + alerts = load_alerts() + rule = alerts["HostMemoryUsageHigh"] + annotations = rule["annotations"] + + assert "HostOutOfMemory" not in alerts + assert rule["labels"]["layer"] == "systemd-{{ $labels.host }}" + assert rule["labels"]["auto_repair"] == "true" + assert "不代表已發生 OOM" in annotations["description"] + assert "oom_kill" in annotations["auto_repair_action"] + assert "drop_caches" not in annotations["auto_repair_action"] + assert "restart" not in annotations["auto_repair_action"] + assert "reboot" not in annotations["auto_repair_action"] + assert "禁止 drop_caches、restart、reboot" in annotations["runbook"] + + +def test_legacy_rule_and_seed_playbook_use_same_read_only_memory_contract() -> None: + legacy_payload = yaml.safe_load(LEGACY_ALERTS.read_text(encoding="utf-8")) + legacy_alerts = { + rule["alert"]: rule + for group in legacy_payload["groups"] + for rule in group.get("rules", []) + if "alert" in rule + } + playbook_source = COLD_START_PLAYBOOKS.read_text(encoding="utf-8") + + assert "HostMemoryUsageHigh" in legacy_alerts + assert "HostOutOfMemory" not in legacy_alerts + assert legacy_alerts["HostMemoryUsageHigh"]["labels"]["layer"] == "systemd-{{ $labels.host }}" + assert "sudo sysctl vm.drop_caches" not in playbook_source + assert '"alert_type": "HostMemoryUsageHigh"' in playbook_source + assert '"HostOutOfMemory"' in playbook_source # historical persisted-event alias only + + def test_backup_aggregate_alert_excludes_old_wrapper_noise() -> None: alerts = load_alerts() expr = str(alerts["BackupAggregateRunFailed"]["expr"])