fix(sre): type backup alerts before generic routing

This commit is contained in:
Your Name
2026-07-19 01:02:09 +08:00
parent dfe75456ab
commit e127c8b56d
4 changed files with 85 additions and 44 deletions

View File

@@ -146,17 +146,15 @@ def classify_alert_early(
規則優先順序(由高到低):
1. ConfigurationDrift / KubeConfigDrift → TYPE-4D (Config Drift 卡片)
2. severity=info/none → TYPE-1 (純資訊,無按鈕)
3. backup/heartbeat 關鍵字 → TYPE-1但 backup failure age > 24h → TYPE-3見下
2. backup/restore/escrow → backup_restore成功 TYPE-1其餘 TYPE-3 唯讀調查)
3. severity=info/none → TYPE-1 (純資訊,無按鈕)
4. Docker/Host/Systemd runner 前綴 → infrastructure/host_resource TYPE-3
5. Kube/Pod/Deploy/Node/Velero/ArgoCD 前綴 → kubernetes TYPE-3
6. Postgres/Redis 前綴 → database TYPE-3
7. 預設 → general TYPE-3
2026-04-25 ogt + Claude Sonnet 4.6 (P0 備份告警升級修復):
- age_hours > 24HostBackupFailed/HostBackupStale/HostBackupMissing 升級為 TYPE-3
原因:備份 25h 未成功是 P0 故障,不是「純資訊」
此時應觸發 LLM 分析 + 自動修復建議,而非靜默發純文字通知
備份失敗不再以 age 延遲分類;立即進入不可寫入的 BackupCheck。
實際 backup/restore/retention/escrow 寫入仍屬 critical break-glass。
C3 修正 (首席架構師 CR 2026-04-13): 從 Router 層 (webhooks.py) 移入 Service 層
原違規: 業務邏輯函數定義在 api/v1/webhooks.py
@@ -220,6 +218,41 @@ def classify_alert_early(
)):
return "business", "TYPE-6B"
# Backup / restore / escrow is a typed data-protection domain. Classify it
# before the generic info shortcut so successful receipts keep their
# domain label, while every non-info failure enters the read-only
# BackupCheck lane immediately instead of spending 24 hours as a generic
# or observation-only alert. This only selects investigation/receipt
# semantics; production backup/restore writes remain critical break-glass.
backup_identity = " ".join(
(
alertname_lower,
str((labels or {}).get("event_type") or "").lower(),
str((labels or {}).get("lane") or "").lower(),
str((labels or {}).get("asset_domain") or "").lower(),
)
)
if any(
marker in backup_identity
for marker in (
"backup",
"escrow",
"restic",
"rclone",
"offsite",
"restore_drill",
"restore-drill",
"restoredrill",
"restore_test",
"restore-test",
"restoretest",
)
):
return (
"backup_restore",
"TYPE-1" if severity in ("info", "none") else "TYPE-3",
)
# 5. 純資訊
if severity in ("info", "none"):
return "info", "TYPE-1"
@@ -230,31 +263,12 @@ def classify_alert_early(
if alertname.startswith("SystemdRunner"):
return "host_resource", "TYPE-3"
# 5. Backup / Heartbeat — 純資訊,不進 LLM
# HostBackupFailed 必須在 Host prefix 前攔截,否則被歸 host_resource/TYPE-3
# 2026-04-12 ogt: 只針對已知主機備份監控 alertname不用寬泛關鍵字
# BackupJobFailed severity=warning 仍走 TYPE-3見測試 test_backup_keyword_warning_not_type1
_BACKUP_TYPE1_NAMES = {
"HostBackupFailed", "HostBackupStale", "HostBackupMissing",
"BackupRestoreTestFailed", "BackupRestoreTestStale",
}
# 2026-04-25 ogt + Claude Sonnet 4.6 (P0 備份告警升級修復):
# 備份失敗 > 24h 不是「純資訊」,是 P0 故障,必須走 TYPE-3 觸發 LLM 分析 + 自動修復
# BackupRestoreTestFailed 屬測試驗證類,不受 age 升級影響(仍 TYPE-1
_BACKUP_AGE_UPGRADE_NAMES = {
"HostBackupFailed", "HostBackupStale", "HostBackupMissing",
}
_BACKUP_AGE_THRESHOLD_HOURS = 24.0
if alertname in _BACKUP_AGE_UPGRADE_NAMES and age_hours > _BACKUP_AGE_THRESHOLD_HOURS:
return "backup_failure", "TYPE-3"
# 5. Heartbeat — 純資訊,不進 LLM
# 2026-04-12 ogt: 補入 DeadMansSwitchHEARTBEAT_ALERT_NAMES 中但之前漏掉)
if (
"watchdog" in alertname_lower
or "deadmansswitch" in alertname_lower
or alertname == "Heartbeat"
or alertname in _BACKUP_TYPE1_NAMES
or alertname.startswith("HostBackup")
):
return "backup", "TYPE-1"