fix(sre): type backup alerts before generic routing
This commit is contained in:
@@ -146,17 +146,15 @@ def classify_alert_early(
|
||||
|
||||
規則優先順序(由高到低):
|
||||
1. ConfigurationDrift / KubeConfigDrift → TYPE-4D (Config Drift 卡片)
|
||||
2. severity=info/none → TYPE-1 (純資訊,無按鈕)
|
||||
3. backup/heartbeat 關鍵字 → TYPE-1(但 backup failure age > 24h → TYPE-3,見下)
|
||||
2. backup/restore/escrow → backup_restore(成功 TYPE-1,其餘 TYPE-3 唯讀調查)
|
||||
3. severity=info/none → TYPE-1 (純資訊,無按鈕)
|
||||
4. Docker/Host/Systemd runner 前綴 → infrastructure/host_resource TYPE-3
|
||||
5. Kube/Pod/Deploy/Node/Velero/ArgoCD 前綴 → kubernetes TYPE-3
|
||||
6. Postgres/Redis 前綴 → database TYPE-3
|
||||
7. 預設 → general TYPE-3
|
||||
|
||||
2026-04-25 ogt + Claude Sonnet 4.6 (P0 備份告警升級修復):
|
||||
- age_hours > 24:HostBackupFailed/HostBackupStale/HostBackupMissing 升級為 TYPE-3
|
||||
原因:備份 25h 未成功是 P0 故障,不是「純資訊」
|
||||
此時應觸發 LLM 分析 + 自動修復建議,而非靜默發純文字通知
|
||||
備份失敗不再以 age 延遲分類;立即進入不可寫入的 BackupCheck。
|
||||
實際 backup/restore/retention/escrow 寫入仍屬 critical break-glass。
|
||||
|
||||
C3 修正 (首席架構師 CR 2026-04-13): 從 Router 層 (webhooks.py) 移入 Service 層
|
||||
原違規: 業務邏輯函數定義在 api/v1/webhooks.py
|
||||
@@ -220,6 +218,41 @@ def classify_alert_early(
|
||||
)):
|
||||
return "business", "TYPE-6B"
|
||||
|
||||
# Backup / restore / escrow is a typed data-protection domain. Classify it
|
||||
# before the generic info shortcut so successful receipts keep their
|
||||
# domain label, while every non-info failure enters the read-only
|
||||
# BackupCheck lane immediately instead of spending 24 hours as a generic
|
||||
# or observation-only alert. This only selects investigation/receipt
|
||||
# semantics; production backup/restore writes remain critical break-glass.
|
||||
backup_identity = " ".join(
|
||||
(
|
||||
alertname_lower,
|
||||
str((labels or {}).get("event_type") or "").lower(),
|
||||
str((labels or {}).get("lane") or "").lower(),
|
||||
str((labels or {}).get("asset_domain") or "").lower(),
|
||||
)
|
||||
)
|
||||
if any(
|
||||
marker in backup_identity
|
||||
for marker in (
|
||||
"backup",
|
||||
"escrow",
|
||||
"restic",
|
||||
"rclone",
|
||||
"offsite",
|
||||
"restore_drill",
|
||||
"restore-drill",
|
||||
"restoredrill",
|
||||
"restore_test",
|
||||
"restore-test",
|
||||
"restoretest",
|
||||
)
|
||||
):
|
||||
return (
|
||||
"backup_restore",
|
||||
"TYPE-1" if severity in ("info", "none") else "TYPE-3",
|
||||
)
|
||||
|
||||
# 5. 純資訊
|
||||
if severity in ("info", "none"):
|
||||
return "info", "TYPE-1"
|
||||
@@ -230,31 +263,12 @@ def classify_alert_early(
|
||||
if alertname.startswith("SystemdRunner"):
|
||||
return "host_resource", "TYPE-3"
|
||||
|
||||
# 5. Backup / Heartbeat — 純資訊,不進 LLM
|
||||
# HostBackupFailed 必須在 Host prefix 前攔截,否則被歸 host_resource/TYPE-3
|
||||
# 2026-04-12 ogt: 只針對已知主機備份監控 alertname,不用寬泛關鍵字
|
||||
# BackupJobFailed severity=warning 仍走 TYPE-3(見測試 test_backup_keyword_warning_not_type1)
|
||||
_BACKUP_TYPE1_NAMES = {
|
||||
"HostBackupFailed", "HostBackupStale", "HostBackupMissing",
|
||||
"BackupRestoreTestFailed", "BackupRestoreTestStale",
|
||||
}
|
||||
# 2026-04-25 ogt + Claude Sonnet 4.6 (P0 備份告警升級修復):
|
||||
# 備份失敗 > 24h 不是「純資訊」,是 P0 故障,必須走 TYPE-3 觸發 LLM 分析 + 自動修復
|
||||
# BackupRestoreTestFailed 屬測試驗證類,不受 age 升級影響(仍 TYPE-1)
|
||||
_BACKUP_AGE_UPGRADE_NAMES = {
|
||||
"HostBackupFailed", "HostBackupStale", "HostBackupMissing",
|
||||
}
|
||||
_BACKUP_AGE_THRESHOLD_HOURS = 24.0
|
||||
if alertname in _BACKUP_AGE_UPGRADE_NAMES and age_hours > _BACKUP_AGE_THRESHOLD_HOURS:
|
||||
return "backup_failure", "TYPE-3"
|
||||
|
||||
# 5. Heartbeat — 純資訊,不進 LLM。
|
||||
# 2026-04-12 ogt: 補入 DeadMansSwitch(HEARTBEAT_ALERT_NAMES 中但之前漏掉)
|
||||
if (
|
||||
"watchdog" in alertname_lower
|
||||
or "deadmansswitch" in alertname_lower
|
||||
or alertname == "Heartbeat"
|
||||
or alertname in _BACKUP_TYPE1_NAMES
|
||||
or alertname.startswith("HostBackup")
|
||||
):
|
||||
return "backup", "TYPE-1"
|
||||
|
||||
|
||||
Reference in New Issue
Block a user