fix(monitoring): classify host memory pressure accurately
This commit is contained in:
@@ -7,6 +7,8 @@ import yaml
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[3]
|
||||
ALERTS = ROOT / "ops" / "monitoring" / "alerts-unified.yml"
|
||||
LEGACY_ALERTS = ROOT / "ops" / "monitoring" / "alerts.yml"
|
||||
COLD_START_PLAYBOOKS = ROOT / "scripts" / "cold_start_playbooks.py"
|
||||
|
||||
|
||||
def load_alerts() -> dict[str, dict]:
|
||||
@@ -54,6 +56,40 @@ def test_critical_sustained_load_alert_uses_deployed_controller_path() -> None:
|
||||
assert "scripts/ops/host-sustained-load-controller.py" not in action
|
||||
|
||||
|
||||
def test_host_memory_pressure_is_not_mislabeled_as_oom_or_host_188() -> None:
|
||||
alerts = load_alerts()
|
||||
rule = alerts["HostMemoryUsageHigh"]
|
||||
annotations = rule["annotations"]
|
||||
|
||||
assert "HostOutOfMemory" not in alerts
|
||||
assert rule["labels"]["layer"] == "systemd-{{ $labels.host }}"
|
||||
assert rule["labels"]["auto_repair"] == "true"
|
||||
assert "不代表已發生 OOM" in annotations["description"]
|
||||
assert "oom_kill" in annotations["auto_repair_action"]
|
||||
assert "drop_caches" not in annotations["auto_repair_action"]
|
||||
assert "restart" not in annotations["auto_repair_action"]
|
||||
assert "reboot" not in annotations["auto_repair_action"]
|
||||
assert "禁止 drop_caches、restart、reboot" in annotations["runbook"]
|
||||
|
||||
|
||||
def test_legacy_rule_and_seed_playbook_use_same_read_only_memory_contract() -> None:
|
||||
legacy_payload = yaml.safe_load(LEGACY_ALERTS.read_text(encoding="utf-8"))
|
||||
legacy_alerts = {
|
||||
rule["alert"]: rule
|
||||
for group in legacy_payload["groups"]
|
||||
for rule in group.get("rules", [])
|
||||
if "alert" in rule
|
||||
}
|
||||
playbook_source = COLD_START_PLAYBOOKS.read_text(encoding="utf-8")
|
||||
|
||||
assert "HostMemoryUsageHigh" in legacy_alerts
|
||||
assert "HostOutOfMemory" not in legacy_alerts
|
||||
assert legacy_alerts["HostMemoryUsageHigh"]["labels"]["layer"] == "systemd-{{ $labels.host }}"
|
||||
assert "sudo sysctl vm.drop_caches" not in playbook_source
|
||||
assert '"alert_type": "HostMemoryUsageHigh"' in playbook_source
|
||||
assert '"HostOutOfMemory"' in playbook_source # historical persisted-event alias only
|
||||
|
||||
|
||||
def test_backup_aggregate_alert_excludes_old_wrapper_noise() -> None:
|
||||
alerts = load_alerts()
|
||||
expr = str(alerts["BackupAggregateRunFailed"]["expr"])
|
||||
|
||||
Reference in New Issue
Block a user