fix(backup): preserve alert receipt identity
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m22s
CD Pipeline / build-and-deploy (push) Successful in 11m52s
AWOOOI Harbor 110 Local Repair / workflow-shape (push) Successful in 0s
AWOOOI Harbor 110 Local Repair / harbor-110-local-repair (push) Successful in 21s
CD Pipeline / post-deploy-checks (push) Successful in 2m29s
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m22s
CD Pipeline / build-and-deploy (push) Successful in 11m52s
AWOOOI Harbor 110 Local Repair / workflow-shape (push) Successful in 0s
AWOOOI Harbor 110 Local Repair / harbor-110-local-repair (push) Successful in 21s
CD Pipeline / post-deploy-checks (push) Successful in 2m29s
This commit is contained in:
@@ -1721,19 +1721,21 @@ groups:
|
||||
runbook: "部署新版 scripts/ops/backup-health-textfile-exporter.py 到 188,刷新 /home/ollama/node_exporter_textfiles/backup_health.prom;不得送測試 Telegram、不得讀 token、不得 restore 到 production。"
|
||||
|
||||
- alert: BackupAlertReceiptStageMissing
|
||||
# Prometheus 會把來源 alertname 覆寫成本規則名稱;先投影為
|
||||
# required_alert 再聚合,避免同一 host 多個 firing alert 產生重複 labelset。
|
||||
expr: |
|
||||
max by (alertname, host) (
|
||||
ALERTS{
|
||||
alertstate="firing",
|
||||
alertname=~"Backup.*|GiteaPrivateBundle.*|GiteaRestoreDrill.*",
|
||||
alertname!="BackupAlertReceiptStageMissing"
|
||||
}
|
||||
)
|
||||
unless on (alertname, host)
|
||||
label_replace(
|
||||
max by (required_alert, host) (awoooi_backup_alert_receipt_fresh == 1),
|
||||
"alertname", "$1", "required_alert", "(.*)"
|
||||
max by (required_alert, host) (
|
||||
label_replace(
|
||||
ALERTS{
|
||||
alertstate="firing",
|
||||
alertname=~"Backup.*|GiteaPrivateBundle.*|GiteaRestoreDrill.*",
|
||||
alertname!="BackupAlertReceiptStageMissing"
|
||||
},
|
||||
"required_alert", "$1", "alertname", "(.*)"
|
||||
)
|
||||
)
|
||||
unless on (required_alert, host)
|
||||
max by (required_alert, host) (awoooi_backup_alert_receipt_fresh == 1)
|
||||
for: 30m
|
||||
labels:
|
||||
severity: warning
|
||||
@@ -1744,9 +1746,9 @@ groups:
|
||||
notification_type: TYPE-1
|
||||
auto_repair: "false"
|
||||
annotations:
|
||||
summary: "{{ $labels.host }} 正在發生的備份告警尚未完成 receipt 閉環"
|
||||
description: "{{ $labels.alertname }} 正在 firing,但尚無新鮮的 Telegram/AwoooP sent/received/dedup/escalation 完整 receipt;未 firing 的規則不再產生假告警。"
|
||||
runbook: "查 awoooi_backup_alert_receipt_stage_fresh{host=\"{{ $labels.host }}\",required_alert=\"{{ $labels.alertname }}\"} 取得缺少階段;只補 timestamp/evidence id,不得寫入 token、chat raw payload 或 secret。"
|
||||
summary: "{{ $labels.host }} 正在發生的備份告警 {{ $labels.required_alert }} 尚未完成 receipt 閉環"
|
||||
description: "{{ $labels.required_alert }} 正在 firing,但尚無新鮮的 Telegram/AwoooP sent/received/dedup/escalation 完整 receipt;未 firing 的規則不再產生假告警。"
|
||||
runbook: "查 awoooi_backup_alert_receipt_stage_fresh{host=\"{{ $labels.host }}\",required_alert=\"{{ $labels.required_alert }}\"} 取得缺少階段;只補 timestamp/evidence id,不得寫入 token、chat raw payload 或 secret。"
|
||||
|
||||
- alert: BackupAggregateRunFailed
|
||||
expr: sum by (host) (awoooi_backup_last_run_failed_count{host="110",exported_job!="backup_all"}) > 0
|
||||
|
||||
45
ops/monitoring/tests/test_backup_alert_receipt_rules.yml
Normal file
45
ops/monitoring/tests/test_backup_alert_receipt_rules.yml
Normal file
@@ -0,0 +1,45 @@
|
||||
rule_files:
|
||||
- ../alerts-unified.yml
|
||||
|
||||
evaluation_interval: 1m
|
||||
|
||||
tests:
|
||||
- name: backup receipt alert preserves each required alert identity
|
||||
interval: 1m
|
||||
input_series:
|
||||
- series: 'ALERTS{alertname="BackupScheduleSingletonMismatch",alertstate="firing",host="110"}'
|
||||
values: '1x40'
|
||||
- series: 'ALERTS{alertname="BackupRetentionPolicyNotLatestOnly",alertstate="firing",host="110"}'
|
||||
values: '1x40'
|
||||
alert_rule_test:
|
||||
- eval_time: 31m
|
||||
alertname: BackupAlertReceiptStageMissing
|
||||
exp_alerts:
|
||||
- exp_labels:
|
||||
alert_category: infrastructure
|
||||
auto_repair: "false"
|
||||
component: backup-alert-receipt
|
||||
host: "110"
|
||||
layer: host-backup
|
||||
notification_type: TYPE-1
|
||||
required_alert: BackupScheduleSingletonMismatch
|
||||
severity: warning
|
||||
team: ops
|
||||
exp_annotations:
|
||||
summary: "110 正在發生的備份告警 BackupScheduleSingletonMismatch 尚未完成 receipt 閉環"
|
||||
description: "BackupScheduleSingletonMismatch 正在 firing,但尚無新鮮的 Telegram/AwoooP sent/received/dedup/escalation 完整 receipt;未 firing 的規則不再產生假告警。"
|
||||
runbook: '查 awoooi_backup_alert_receipt_stage_fresh{host="110",required_alert="BackupScheduleSingletonMismatch"} 取得缺少階段;只補 timestamp/evidence id,不得寫入 token、chat raw payload 或 secret。'
|
||||
- exp_labels:
|
||||
alert_category: infrastructure
|
||||
auto_repair: "false"
|
||||
component: backup-alert-receipt
|
||||
host: "110"
|
||||
layer: host-backup
|
||||
notification_type: TYPE-1
|
||||
required_alert: BackupRetentionPolicyNotLatestOnly
|
||||
severity: warning
|
||||
team: ops
|
||||
exp_annotations:
|
||||
summary: "110 正在發生的備份告警 BackupRetentionPolicyNotLatestOnly 尚未完成 receipt 閉環"
|
||||
description: "BackupRetentionPolicyNotLatestOnly 正在 firing,但尚無新鮮的 Telegram/AwoooP sent/received/dedup/escalation 完整 receipt;未 firing 的規則不再產生假告警。"
|
||||
runbook: '查 awoooi_backup_alert_receipt_stage_fresh{host="110",required_alert="BackupRetentionPolicyNotLatestOnly"} 取得缺少階段;只補 timestamp/evidence id,不得寫入 token、chat raw payload 或 secret。'
|
||||
Reference in New Issue
Block a user