fix(monitoring): classify host memory pressure accurately
This commit is contained in:
@@ -104,18 +104,18 @@ PLAYBOOK_TEMPLATES = [
|
||||
"severity_range": ["P2", "P3"],
|
||||
},
|
||||
{
|
||||
"name": "節點記憶體不足 — 清理 Cache",
|
||||
"alert_type": "HostOutOfMemory",
|
||||
"description": "主機記憶體不足,清理 Page Cache 釋放空間",
|
||||
"name": "節點記憶體壓力 — 唯讀診斷",
|
||||
"alert_type": "HostMemoryUsageHigh",
|
||||
"description": "主機 MemAvailable 偏低;先驗證 OOM、swap、程序與服務狀態,不做破壞性快取清理",
|
||||
"category": "infrastructure",
|
||||
"repair_steps": [
|
||||
{"command": "ssh {target} 'free -h'", "purpose": "確認記憶體狀況"},
|
||||
{"command": "ssh {target} 'sudo sync && sudo sysctl vm.drop_caches=3'", "purpose": "清理 Page Cache"},
|
||||
{"command": "ssh {target} 'free -h'", "purpose": "確認記憶體已釋放"},
|
||||
{"command": "ssh {target} 'free -h'", "purpose": "讀取 MemAvailable 與 swap 狀況"},
|
||||
{"command": "ssh {target} \"awk '$1 == \\\"oom_kill\\\" { print }' /proc/vmstat\"", "purpose": "確認 kernel OOM kill 累計值"},
|
||||
{"command": "ssh {target} 'ps aux --sort=-%mem | head -20'", "purpose": "讀取主要記憶體程序"},
|
||||
],
|
||||
"estimated_minutes": 5,
|
||||
"symptom_alertnames": ["HostOutOfMemory", "NodeMemoryPressure"],
|
||||
"severity_range": ["P1", "P2"],
|
||||
"symptom_alertnames": ["HostMemoryUsageHigh", "HostOutOfMemory", "NodeMemoryPressure"],
|
||||
"severity_range": ["P2", "P3"],
|
||||
},
|
||||
{
|
||||
"name": "磁碟使用率過高 — 清理舊日誌",
|
||||
|
||||
@@ -7,6 +7,8 @@ import yaml
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[3]
|
||||
ALERTS = ROOT / "ops" / "monitoring" / "alerts-unified.yml"
|
||||
LEGACY_ALERTS = ROOT / "ops" / "monitoring" / "alerts.yml"
|
||||
COLD_START_PLAYBOOKS = ROOT / "scripts" / "cold_start_playbooks.py"
|
||||
|
||||
|
||||
def load_alerts() -> dict[str, dict]:
|
||||
@@ -54,6 +56,40 @@ def test_critical_sustained_load_alert_uses_deployed_controller_path() -> None:
|
||||
assert "scripts/ops/host-sustained-load-controller.py" not in action
|
||||
|
||||
|
||||
def test_host_memory_pressure_is_not_mislabeled_as_oom_or_host_188() -> None:
|
||||
alerts = load_alerts()
|
||||
rule = alerts["HostMemoryUsageHigh"]
|
||||
annotations = rule["annotations"]
|
||||
|
||||
assert "HostOutOfMemory" not in alerts
|
||||
assert rule["labels"]["layer"] == "systemd-{{ $labels.host }}"
|
||||
assert rule["labels"]["auto_repair"] == "true"
|
||||
assert "不代表已發生 OOM" in annotations["description"]
|
||||
assert "oom_kill" in annotations["auto_repair_action"]
|
||||
assert "drop_caches" not in annotations["auto_repair_action"]
|
||||
assert "restart" not in annotations["auto_repair_action"]
|
||||
assert "reboot" not in annotations["auto_repair_action"]
|
||||
assert "禁止 drop_caches、restart、reboot" in annotations["runbook"]
|
||||
|
||||
|
||||
def test_legacy_rule_and_seed_playbook_use_same_read_only_memory_contract() -> None:
|
||||
legacy_payload = yaml.safe_load(LEGACY_ALERTS.read_text(encoding="utf-8"))
|
||||
legacy_alerts = {
|
||||
rule["alert"]: rule
|
||||
for group in legacy_payload["groups"]
|
||||
for rule in group.get("rules", [])
|
||||
if "alert" in rule
|
||||
}
|
||||
playbook_source = COLD_START_PLAYBOOKS.read_text(encoding="utf-8")
|
||||
|
||||
assert "HostMemoryUsageHigh" in legacy_alerts
|
||||
assert "HostOutOfMemory" not in legacy_alerts
|
||||
assert legacy_alerts["HostMemoryUsageHigh"]["labels"]["layer"] == "systemd-{{ $labels.host }}"
|
||||
assert "sudo sysctl vm.drop_caches" not in playbook_source
|
||||
assert '"alert_type": "HostMemoryUsageHigh"' in playbook_source
|
||||
assert '"HostOutOfMemory"' in playbook_source # historical persisted-event alias only
|
||||
|
||||
|
||||
def test_backup_aggregate_alert_excludes_old_wrapper_noise() -> None:
|
||||
alerts = load_alerts()
|
||||
expr = str(alerts["BackupAggregateRunFailed"]["expr"])
|
||||
|
||||
Reference in New Issue
Block a user