fix(monitoring): classify host memory pressure accurately

This commit is contained in:
ogt
2026-07-15 06:15:13 +08:00
parent 89595a7d8d
commit 5c0c5f6da2
11 changed files with 76 additions and 25 deletions

View File

@@ -7,6 +7,8 @@ import yaml
ROOT = Path(__file__).resolve().parents[3]
ALERTS = ROOT / "ops" / "monitoring" / "alerts-unified.yml"
LEGACY_ALERTS = ROOT / "ops" / "monitoring" / "alerts.yml"
COLD_START_PLAYBOOKS = ROOT / "scripts" / "cold_start_playbooks.py"
def load_alerts() -> dict[str, dict]:
@@ -54,6 +56,40 @@ def test_critical_sustained_load_alert_uses_deployed_controller_path() -> None:
assert "scripts/ops/host-sustained-load-controller.py" not in action
def test_host_memory_pressure_is_not_mislabeled_as_oom_or_host_188() -> None:
alerts = load_alerts()
rule = alerts["HostMemoryUsageHigh"]
annotations = rule["annotations"]
assert "HostOutOfMemory" not in alerts
assert rule["labels"]["layer"] == "systemd-{{ $labels.host }}"
assert rule["labels"]["auto_repair"] == "true"
assert "不代表已發生 OOM" in annotations["description"]
assert "oom_kill" in annotations["auto_repair_action"]
assert "drop_caches" not in annotations["auto_repair_action"]
assert "restart" not in annotations["auto_repair_action"]
assert "reboot" not in annotations["auto_repair_action"]
assert "禁止 drop_caches、restart、reboot" in annotations["runbook"]
def test_legacy_rule_and_seed_playbook_use_same_read_only_memory_contract() -> None:
legacy_payload = yaml.safe_load(LEGACY_ALERTS.read_text(encoding="utf-8"))
legacy_alerts = {
rule["alert"]: rule
for group in legacy_payload["groups"]
for rule in group.get("rules", [])
if "alert" in rule
}
playbook_source = COLD_START_PLAYBOOKS.read_text(encoding="utf-8")
assert "HostMemoryUsageHigh" in legacy_alerts
assert "HostOutOfMemory" not in legacy_alerts
assert legacy_alerts["HostMemoryUsageHigh"]["labels"]["layer"] == "systemd-{{ $labels.host }}"
assert "sudo sysctl vm.drop_caches" not in playbook_source
assert '"alert_type": "HostMemoryUsageHigh"' in playbook_source
assert '"HostOutOfMemory"' in playbook_source # historical persisted-event alias only
def test_backup_aggregate_alert_excludes_old_wrapper_noise() -> None:
alerts = load_alerts()
expr = str(alerts["BackupAggregateRunFailed"]["expr"])