From e127c8b56d5089ca5fbd2d737b57a060fed3bc9c Mon Sep 17 00:00:00 2001 From: Your Name Date: Sun, 19 Jul 2026 01:02:09 +0800 Subject: [PATCH] fix(sre): type backup alerts before generic routing --- apps/api/src/services/incident_service.py | 66 +++++++++++-------- ...est_backup_restore_alertmanager_ingress.py | 8 +-- apps/api/tests/test_classify_alert_early.py | 41 ++++++++++-- apps/api/tests/test_golden_regression.py | 14 ++-- 4 files changed, 85 insertions(+), 44 deletions(-) diff --git a/apps/api/src/services/incident_service.py b/apps/api/src/services/incident_service.py index 0b9ea02a4..ce01fa93b 100644 --- a/apps/api/src/services/incident_service.py +++ b/apps/api/src/services/incident_service.py @@ -146,17 +146,15 @@ def classify_alert_early( 規則優先順序（由高到低）: 1. ConfigurationDrift / KubeConfigDrift → TYPE-4D (Config Drift 卡片) - 2. severity=info/none → TYPE-1 (純資訊，無按鈕) - 3. backup/heartbeat 關鍵字 → TYPE-1（但 backup failure age > 24h → TYPE-3，見下） + 2. backup/restore/escrow → backup_restore（成功 TYPE-1，其餘 TYPE-3 唯讀調查） + 3. severity=info/none → TYPE-1 (純資訊，無按鈕) 4. Docker/Host/Systemd runner 前綴 → infrastructure/host_resource TYPE-3 5. Kube/Pod/Deploy/Node/Velero/ArgoCD 前綴 → kubernetes TYPE-3 6. Postgres/Redis 前綴 → database TYPE-3 7. 預設 → general TYPE-3 - 2026-04-25 ogt + Claude Sonnet 4.6 (P0 備份告警升級修復): - - age_hours > 24：HostBackupFailed/HostBackupStale/HostBackupMissing 升級為 TYPE-3 - 原因：備份 25h 未成功是 P0 故障，不是「純資訊」 - 此時應觸發 LLM 分析 + 自動修復建議，而非靜默發純文字通知 + 備份失敗不再以 age 延遲分類；立即進入不可寫入的 BackupCheck。 + 實際 backup/restore/retention/escrow 寫入仍屬 critical break-glass。 C3 修正 (首席架構師 CR 2026-04-13): 從 Router 層 (webhooks.py) 移入 Service 層 原違規: 業務邏輯函數定義在 api/v1/webhooks.py @@ -220,6 +218,41 @@ def classify_alert_early( )): return "business", "TYPE-6B" + # Backup / restore / escrow is a typed data-protection domain. Classify it + # before the generic info shortcut so successful receipts keep their + # domain label, while every non-info failure enters the read-only + # BackupCheck lane immediately instead of spending 24 hours as a generic + # or observation-only alert. This only selects investigation/receipt + # semantics; production backup/restore writes remain critical break-glass. + backup_identity = " ".join( + ( + alertname_lower, + str((labels or {}).get("event_type") or "").lower(), + str((labels or {}).get("lane") or "").lower(), + str((labels or {}).get("asset_domain") or "").lower(), + ) + ) + if any( + marker in backup_identity + for marker in ( + "backup", + "escrow", + "restic", + "rclone", + "offsite", + "restore_drill", + "restore-drill", + "restoredrill", + "restore_test", + "restore-test", + "restoretest", + ) + ): + return ( + "backup_restore", + "TYPE-1" if severity in ("info", "none") else "TYPE-3", + ) + # 5. 純資訊 if severity in ("info", "none"): return "info", "TYPE-1" @@ -230,31 +263,12 @@ def classify_alert_early( if alertname.startswith("SystemdRunner"): return "host_resource", "TYPE-3" - # 5. Backup / Heartbeat — 純資訊，不進 LLM - # HostBackupFailed 必須在 Host prefix 前攔截，否則被歸 host_resource/TYPE-3 - # 2026-04-12 ogt: 只針對已知主機備份監控 alertname，不用寬泛關鍵字 - # BackupJobFailed severity=warning 仍走 TYPE-3（見測試 test_backup_keyword_warning_not_type1） - _BACKUP_TYPE1_NAMES = { - "HostBackupFailed", "HostBackupStale", "HostBackupMissing", - "BackupRestoreTestFailed", "BackupRestoreTestStale", - } - # 2026-04-25 ogt + Claude Sonnet 4.6 (P0 備份告警升級修復): - # 備份失敗 > 24h 不是「純資訊」，是 P0 故障，必須走 TYPE-3 觸發 LLM 分析 + 自動修復 - # BackupRestoreTestFailed 屬測試驗證類，不受 age 升級影響（仍 TYPE-1） - _BACKUP_AGE_UPGRADE_NAMES = { - "HostBackupFailed", "HostBackupStale", "HostBackupMissing", - } - _BACKUP_AGE_THRESHOLD_HOURS = 24.0 - if alertname in _BACKUP_AGE_UPGRADE_NAMES and age_hours > _BACKUP_AGE_THRESHOLD_HOURS: - return "backup_failure", "TYPE-3" - + # 5. Heartbeat — 純資訊，不進 LLM。 # 2026-04-12 ogt: 補入 DeadMansSwitch（HEARTBEAT_ALERT_NAMES 中但之前漏掉） if ( "watchdog" in alertname_lower or "deadmansswitch" in alertname_lower or alertname == "Heartbeat" - or alertname in _BACKUP_TYPE1_NAMES - or alertname.startswith("HostBackup") ): return "backup", "TYPE-1" diff --git a/apps/api/tests/test_backup_restore_alertmanager_ingress.py b/apps/api/tests/test_backup_restore_alertmanager_ingress.py index d63468c08..8c4331c7f 100644 --- a/apps/api/tests/test_backup_restore_alertmanager_ingress.py +++ b/apps/api/tests/test_backup_restore_alertmanager_ingress.py @@ -336,7 +336,7 @@ async def test_backup_source_only_v2_lookup_reuses_only_full_legacy_owner() -> N @pytest.mark.asyncio -async def test_alertmanager_type1_backup_queues_durable_owner_before_info_shortcut( +async def test_alertmanager_backup_failure_queues_typed_durable_owner_before_generic_flow( monkeypatch: pytest.MonkeyPatch, ) -> None: grouped = SimpleNamespace(is_grouped=False) @@ -360,7 +360,7 @@ async def test_alertmanager_type1_backup_queues_durable_owner_before_info_shortc webhooks, "get_approval_service", lambda: (_ for _ in ()).throw( - AssertionError("generic approval/TYPE-1 branch must not run") + AssertionError("generic approval branch must not run") ), ) @@ -397,8 +397,8 @@ async def test_alertmanager_type1_backup_queues_durable_owner_before_info_shortc call = process.await_args.kwargs assert call["alertname"] == "HostBackupFailed" assert call["source_event_fingerprint"] == "native-backup-fingerprint" - assert call["notification_type"] == "TYPE-1" - assert call["alert_category"] == "backup" + assert call["notification_type"] == "TYPE-3" + assert call["alert_category"] == "backup_restore" assert call["target_resource"] == "backup_restore" diff --git a/apps/api/tests/test_classify_alert_early.py b/apps/api/tests/test_classify_alert_early.py index 533bf220a..01be8e8eb 100644 --- a/apps/api/tests/test_classify_alert_early.py +++ b/apps/api/tests/test_classify_alert_early.py @@ -63,15 +63,43 @@ class TestInfoAlerts: assert ac == "info" def test_backup_keyword_info_only(self): - # severity=info → severity 規則先命中，TYPE-1 + # 成功通知仍是 TYPE-1，但不可丟失備份領域分類。 ac, nt = classify_alert_early("BackupJobComplete", "info", {}) + assert ac == "backup_restore" assert nt == "TYPE-1" def test_backup_keyword_warning_not_type1(self): - # BackupJobFailed severity=warning → 繼續走 prefix 規則，不應是 TYPE-1 + # 備份失敗立即進唯讀 BackupCheck，不等 24 小時。 ac, nt = classify_alert_early("BackupJobFailed", "warning", {}) + assert ac == "backup_restore" assert nt == "TYPE-3" + @pytest.mark.parametrize( + "alertname", + ( + "Backup.awoooi-frequent", + "Backup.langfuse", + "Backup.signoz", + "HostBackupFailed", + "BackupCredentialEscrowEvidenceMissing", + ), + ) + def test_backup_products_share_typed_readback_domain(self, alertname): + ac, nt = classify_alert_early(alertname, "warning", {}) + assert (ac, nt) == ("backup_restore", "TYPE-3") + + def test_backup_domain_label_routes_even_without_backup_alertname(self): + ac, nt = classify_alert_early( + "DataProtectionEvidenceMissing", + "critical", + {"asset_domain": "backup_restore"}, + ) + assert (ac, nt) == ("backup_restore", "TYPE-3") + + def test_generic_service_restored_is_not_a_backup_identity(self): + ac, nt = classify_alert_early("ServiceRestored", "info", {}) + assert (ac, nt) == ("info", "TYPE-1") + def test_watchdog_heartbeat(self): # Watchdog (Alertmanager 心跳) severity=none → severity 規則先命中，TYPE-1 ac, nt = classify_alert_early("Watchdog", "none", {}) @@ -84,9 +112,9 @@ class TestInfoAlerts: assert nt == "TYPE-1" def test_backup_critical_not_type1(self): - # critical backup 告警應走各自 prefix，不是純資訊 + # critical backup 告警必須是明確的備份領域唯讀調查。 ac, nt = classify_alert_early("BACKUP_MISSING", "critical", {}) - assert nt == "TYPE-3" + assert (ac, nt) == ("backup_restore", "TYPE-3") # --------------------------------------------------------------------------- # @@ -270,10 +298,9 @@ class TestKubernetes: assert ac == "kubernetes" def test_velero_backup_failed_is_kubernetes(self): - # VeleroBackupFailed severity=critical → backup 規則不命中，走 Velero prefix → kubernetes TYPE-3 + # Velero 備份是資料保護事件，不可誤掉到 K8s 寫入 executor。 ac, nt = classify_alert_early("VeleroBackupFailed", "critical", {}) - assert nt == "TYPE-3" - assert ac == "kubernetes" + assert (ac, nt) == ("backup_restore", "TYPE-3") def test_velero_backup_success_info_is_type1(self): # VeleroBackupSuccess severity=info → TYPE-1 diff --git a/apps/api/tests/test_golden_regression.py b/apps/api/tests/test_golden_regression.py index acccb8488..f441f9f7e 100644 --- a/apps/api/tests/test_golden_regression.py +++ b/apps/api/tests/test_golden_regression.py @@ -66,23 +66,23 @@ class TestHighCpuVariants: class TestBackup: def test_host_backup_failed_fresh(self): - """< 24h 備份失敗 → TYPE-1（pure info）""" + """< 24h 備份失敗也要立即進唯讀 BackupCheck。""" cat, ntype = classify_alert_early("HostBackupFailed", "warning", age_hours=10.0) - assert cat == "backup" and ntype == "TYPE-1" + assert cat == "backup_restore" and ntype == "TYPE-3" def test_host_backup_failed_stale_upgrade(self): - """> 24h 備份失敗 → 升級為 TYPE-3（P0 故障）""" + """> 24h 仍保持同一 typed backup/restore lane。""" cat, ntype = classify_alert_early("HostBackupFailed", "warning", age_hours=25.0) - assert cat == "backup_failure" and ntype == "TYPE-3" + assert cat == "backup_restore" and ntype == "TYPE-3" def test_host_backup_stale_upgrade(self): cat, ntype = classify_alert_early("HostBackupStale", "warning", age_hours=30.0) - assert cat == "backup_failure" and ntype == "TYPE-3" + assert cat == "backup_restore" and ntype == "TYPE-3" def test_backup_restore_test_not_upgraded(self): - """BackupRestoreTestFailed 不受 age 升級影響""" + """Restore drill 失敗也是唯讀調查，不誤標資訊。""" cat, ntype = classify_alert_early("BackupRestoreTestFailed", "warning", age_hours=48.0) - assert ntype == "TYPE-1" + assert cat == "backup_restore" and ntype == "TYPE-3" class TestDatabase: