wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 2 Packages Projects Releases Wiki Activity

fix(sre): type backup alerts before generic routing

Browse Source
This commit is contained in:
Your Name
2026-07-19 01:02:09 +08:00
parent dfe75456ab
commit e127c8b56d
4 changed files with 85 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

66
apps/api/src/services/incident_service.py
View File

@@ -146,17 +146,15 @@ def classify_alert_early(
規則優先順序（由高到低）:
1. ConfigurationDrift / KubeConfigDrift → TYPE-4D (Config Drift 卡片)
2. severity=info/none → TYPE-1 (純資訊，無按鈕)
3. backup/heartbeat 關鍵字 → TYPE-1（但 backup failure age > 24h → TYPE-3，見下）
2. backup/restore/escrow → backup_restore（成功 TYPE-1，其餘 TYPE-3 唯讀調查）
3. severity=info/none → TYPE-1 (純資訊，無按鈕)
4. Docker/Host/Systemd runner 前綴 → infrastructure/host_resource TYPE-3
5. Kube/Pod/Deploy/Node/Velero/ArgoCD 前綴 → kubernetes TYPE-3
6. Postgres/Redis 前綴 → database TYPE-3
7. 預設 → general TYPE-3
2026-04-25 ogt + Claude Sonnet 4.6 (P0 備份告警升級修復):
- age_hours > 24：HostBackupFailed/HostBackupStale/HostBackupMissing 升級為 TYPE-3
原因：備份 25h 未成功是 P0 故障，不是「純資訊」
此時應觸發 LLM 分析 + 自動修復建議，而非靜默發純文字通知
備份失敗不再以 age 延遲分類；立即進入不可寫入的 BackupCheck。
實際 backup/restore/retention/escrow 寫入仍屬 critical break-glass。
C3 修正 (首席架構師 CR 2026-04-13): 從 Router 層 (webhooks.py) 移入 Service 層
原違規: 業務邏輯函數定義在 api/v1/webhooks.py
@@ -220,6 +218,41 @@ def classify_alert_early(
)):
return "business", "TYPE-6B"
# Backup / restore / escrow is a typed data-protection domain. Classify it
# before the generic info shortcut so successful receipts keep their
# domain label, while every non-info failure enters the read-only
# BackupCheck lane immediately instead of spending 24 hours as a generic
# or observation-only alert. This only selects investigation/receipt
# semantics; production backup/restore writes remain critical break-glass.
backup_identity = " ".join(
(
alertname_lower,
str((labels or {}).get("event_type") or "").lower(),
str((labels or {}).get("lane") or "").lower(),
str((labels or {}).get("asset_domain") or "").lower(),
)
)
if any(
marker in backup_identity
for marker in (
"backup",
"escrow",
"restic",
"rclone",
"offsite",
"restore_drill",
"restore-drill",
"restoredrill",
"restore_test",
"restore-test",
"restoretest",
)
):
return (
"backup_restore",
"TYPE-1" if severity in ("info", "none") else "TYPE-3",
)
# 5. 純資訊
if severity in ("info", "none"):
return "info", "TYPE-1"
@@ -230,31 +263,12 @@ def classify_alert_early(
if alertname.startswith("SystemdRunner"):
return "host_resource", "TYPE-3"
# 5. Backup / Heartbeat — 純資訊，不進 LLM
# HostBackupFailed 必須在 Host prefix 前攔截，否則被歸 host_resource/TYPE-3
# 2026-04-12 ogt: 只針對已知主機備份監控 alertname，不用寬泛關鍵字
# BackupJobFailed severity=warning 仍走 TYPE-3（見測試 test_backup_keyword_warning_not_type1）
_BACKUP_TYPE1_NAMES = {
"HostBackupFailed", "HostBackupStale", "HostBackupMissing",
"BackupRestoreTestFailed", "BackupRestoreTestStale",
}
# 2026-04-25 ogt + Claude Sonnet 4.6 (P0 備份告警升級修復):
# 備份失敗 > 24h 不是「純資訊」，是 P0 故障，必須走 TYPE-3 觸發 LLM 分析 + 自動修復
# BackupRestoreTestFailed 屬測試驗證類，不受 age 升級影響（仍 TYPE-1）
_BACKUP_AGE_UPGRADE_NAMES = {
"HostBackupFailed", "HostBackupStale", "HostBackupMissing",
}
_BACKUP_AGE_THRESHOLD_HOURS = 24.0
if alertname in _BACKUP_AGE_UPGRADE_NAMES and age_hours > _BACKUP_AGE_THRESHOLD_HOURS:
return "backup_failure", "TYPE-3"
# 5. Heartbeat — 純資訊，不進 LLM。
# 2026-04-12 ogt: 補入 DeadMansSwitch（HEARTBEAT_ALERT_NAMES 中但之前漏掉）
if (
"watchdog" in alertname_lower
or "deadmansswitch" in alertname_lower
or alertname == "Heartbeat"
or alertname in _BACKUP_TYPE1_NAMES
or alertname.startswith("HostBackup")
):
return "backup", "TYPE-1"

8
apps/api/tests/test_backup_restore_alertmanager_ingress.py
View File

@@ -336,7 +336,7 @@ async def test_backup_source_only_v2_lookup_reuses_only_full_legacy_owner() -> N
@pytest.mark.asyncio
async def test_alertmanager_type1_backup_queues_durable_owner_before_info_shortcut(
async def test_alertmanager_backup_failure_queues_typed_durable_owner_before_generic_flow(
monkeypatch: pytest.MonkeyPatch,
) -> None:
grouped = SimpleNamespace(is_grouped=False)
@@ -360,7 +360,7 @@ async def test_alertmanager_type1_backup_queues_durable_owner_before_info_shortc
webhooks,
"get_approval_service",
lambda: (_ for _ in ()).throw(
AssertionError("generic approval/TYPE-1 branch must not run")
AssertionError("generic approval branch must not run")
),
)
@@ -397,8 +397,8 @@ async def test_alertmanager_type1_backup_queues_durable_owner_before_info_shortc
call = process.await_args.kwargs
assert call["alertname"] == "HostBackupFailed"
assert call["source_event_fingerprint"] == "native-backup-fingerprint"
assert call["notification_type"] == "TYPE-1"
assert call["alert_category"] == "backup"
assert call["notification_type"] == "TYPE-3"
assert call["alert_category"] == "backup_restore"
assert call["target_resource"] == "backup_restore"

41
apps/api/tests/test_classify_alert_early.py
View File

@@ -63,15 +63,43 @@ class TestInfoAlerts:
assert ac == "info"
def test_backup_keyword_info_only(self):
# severity=info → severity 規則先命中，TYPE-1
# 成功通知仍是 TYPE-1，但不可丟失備份領域分類。
ac, nt = classify_alert_early("BackupJobComplete", "info", {})
assert ac == "backup_restore"
assert nt == "TYPE-1"
def test_backup_keyword_warning_not_type1(self):
# BackupJobFailed severity=warning → 繼續走 prefix 規則，不應是 TYPE-1
# 備份失敗立即進唯讀 BackupCheck，不等 24 小時。
ac, nt = classify_alert_early("BackupJobFailed", "warning", {})
assert ac == "backup_restore"
assert nt == "TYPE-3"
@pytest.mark.parametrize(
"alertname",
(
"Backup.awoooi-frequent",
"Backup.langfuse",
"Backup.signoz",
"HostBackupFailed",
"BackupCredentialEscrowEvidenceMissing",
),
)
def test_backup_products_share_typed_readback_domain(self, alertname):
ac, nt = classify_alert_early(alertname, "warning", {})
assert (ac, nt) == ("backup_restore", "TYPE-3")
def test_backup_domain_label_routes_even_without_backup_alertname(self):
ac, nt = classify_alert_early(
"DataProtectionEvidenceMissing",
"critical",
{"asset_domain": "backup_restore"},
)
assert (ac, nt) == ("backup_restore", "TYPE-3")
def test_generic_service_restored_is_not_a_backup_identity(self):
ac, nt = classify_alert_early("ServiceRestored", "info", {})
assert (ac, nt) == ("info", "TYPE-1")
def test_watchdog_heartbeat(self):
# Watchdog (Alertmanager 心跳) severity=none → severity 規則先命中，TYPE-1
ac, nt = classify_alert_early("Watchdog", "none", {})
@@ -84,9 +112,9 @@ class TestInfoAlerts:
assert nt == "TYPE-1"
def test_backup_critical_not_type1(self):
# critical backup 告警應走各自 prefix，不是純資訊
# critical backup 告警必須是明確的備份領域唯讀調查。
ac, nt = classify_alert_early("BACKUP_MISSING", "critical", {})
assert nt == "TYPE-3"
assert (ac, nt) == ("backup_restore", "TYPE-3")
# --------------------------------------------------------------------------- #
@@ -270,10 +298,9 @@ class TestKubernetes:
assert ac == "kubernetes"
def test_velero_backup_failed_is_kubernetes(self):
# VeleroBackupFailed severity=critical → backup 規則不命中，走 Velero prefix → kubernetes TYPE-3
# Velero 備份是資料保護事件，不可誤掉到 K8s 寫入 executor。
ac, nt = classify_alert_early("VeleroBackupFailed", "critical", {})
assert nt == "TYPE-3"
assert ac == "kubernetes"
assert (ac, nt) == ("backup_restore", "TYPE-3")
def test_velero_backup_success_info_is_type1(self):
# VeleroBackupSuccess severity=info → TYPE-1

14
apps/api/tests/test_golden_regression.py
View File

@@ -66,23 +66,23 @@ class TestHighCpuVariants:
class TestBackup:
def test_host_backup_failed_fresh(self):
"""< 24h 備份失敗 → TYPE-1（pure info）"""
"""< 24h 備份失敗也要立即進唯讀 BackupCheck。"""
cat, ntype = classify_alert_early("HostBackupFailed", "warning", age_hours=10.0)
assert cat == "backup" and ntype == "TYPE-1"
assert cat == "backup_restore" and ntype == "TYPE-3"
def test_host_backup_failed_stale_upgrade(self):
"""> 24h 備份失敗 → 升級為 TYPE-3（P0 故障）"""
"""> 24h 仍保持同一 typed backup/restore lane。"""
cat, ntype = classify_alert_early("HostBackupFailed", "warning", age_hours=25.0)
assert cat == "backup_failure" and ntype == "TYPE-3"
assert cat == "backup_restore" and ntype == "TYPE-3"
def test_host_backup_stale_upgrade(self):
cat, ntype = classify_alert_early("HostBackupStale", "warning", age_hours=30.0)
assert cat == "backup_failure" and ntype == "TYPE-3"
assert cat == "backup_restore" and ntype == "TYPE-3"
def test_backup_restore_test_not_upgraded(self):
"""BackupRestoreTestFailed 不受 age 升級影響"""
"""Restore drill 失敗也是唯讀調查，不誤標資訊。"""
cat, ntype = classify_alert_early("BackupRestoreTestFailed", "warning", age_hours=48.0)
assert ntype == "TYPE-1"
assert cat == "backup_restore" and ntype == "TYPE-3"
class TestDatabase: