fix(backup): preserve alert receipt identity
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m22s
CD Pipeline / build-and-deploy (push) Successful in 11m52s
AWOOOI Harbor 110 Local Repair / workflow-shape (push) Successful in 0s
AWOOOI Harbor 110 Local Repair / harbor-110-local-repair (push) Successful in 21s
CD Pipeline / post-deploy-checks (push) Successful in 2m29s

This commit is contained in:
ogt
2026-07-14 23:34:55 +08:00
parent 63b241b71e
commit a535fa652f
10 changed files with 315 additions and 34 deletions

View File

@@ -1721,19 +1721,21 @@ groups:
runbook: "部署新版 scripts/ops/backup-health-textfile-exporter.py 到 188刷新 /home/ollama/node_exporter_textfiles/backup_health.prom不得送測試 Telegram、不得讀 token、不得 restore 到 production。"
- alert: BackupAlertReceiptStageMissing
# Prometheus 會把來源 alertname 覆寫成本規則名稱;先投影為
# required_alert 再聚合,避免同一 host 多個 firing alert 產生重複 labelset。
expr: |
max by (alertname, host) (
ALERTS{
alertstate="firing",
alertname=~"Backup.*|GiteaPrivateBundle.*|GiteaRestoreDrill.*",
alertname!="BackupAlertReceiptStageMissing"
}
)
unless on (alertname, host)
label_replace(
max by (required_alert, host) (awoooi_backup_alert_receipt_fresh == 1),
"alertname", "$1", "required_alert", "(.*)"
max by (required_alert, host) (
label_replace(
ALERTS{
alertstate="firing",
alertname=~"Backup.*|GiteaPrivateBundle.*|GiteaRestoreDrill.*",
alertname!="BackupAlertReceiptStageMissing"
},
"required_alert", "$1", "alertname", "(.*)"
)
)
unless on (required_alert, host)
max by (required_alert, host) (awoooi_backup_alert_receipt_fresh == 1)
for: 30m
labels:
severity: warning
@@ -1744,9 +1746,9 @@ groups:
notification_type: TYPE-1
auto_repair: "false"
annotations:
summary: "{{ $labels.host }} 正在發生的備份告警尚未完成 receipt 閉環"
description: "{{ $labels.alertname }} 正在 firing但尚無新鮮的 Telegram/AwoooP sent/received/dedup/escalation 完整 receipt未 firing 的規則不再產生假告警。"
runbook: "查 awoooi_backup_alert_receipt_stage_fresh{host=\"{{ $labels.host }}\",required_alert=\"{{ $labels.alertname }}\"} 取得缺少階段;只補 timestamp/evidence id不得寫入 token、chat raw payload 或 secret。"
summary: "{{ $labels.host }} 正在發生的備份告警 {{ $labels.required_alert }} 尚未完成 receipt 閉環"
description: "{{ $labels.required_alert }} 正在 firing但尚無新鮮的 Telegram/AwoooP sent/received/dedup/escalation 完整 receipt未 firing 的規則不再產生假告警。"
runbook: "查 awoooi_backup_alert_receipt_stage_fresh{host=\"{{ $labels.host }}\",required_alert=\"{{ $labels.required_alert }}\"} 取得缺少階段;只補 timestamp/evidence id不得寫入 token、chat raw payload 或 secret。"
- alert: BackupAggregateRunFailed
expr: sum by (host) (awoooi_backup_last_run_failed_count{host="110",exported_job!="backup_all"}) > 0

View File

@@ -0,0 +1,45 @@
rule_files:
- ../alerts-unified.yml
evaluation_interval: 1m
tests:
- name: backup receipt alert preserves each required alert identity
interval: 1m
input_series:
- series: 'ALERTS{alertname="BackupScheduleSingletonMismatch",alertstate="firing",host="110"}'
values: '1x40'
- series: 'ALERTS{alertname="BackupRetentionPolicyNotLatestOnly",alertstate="firing",host="110"}'
values: '1x40'
alert_rule_test:
- eval_time: 31m
alertname: BackupAlertReceiptStageMissing
exp_alerts:
- exp_labels:
alert_category: infrastructure
auto_repair: "false"
component: backup-alert-receipt
host: "110"
layer: host-backup
notification_type: TYPE-1
required_alert: BackupScheduleSingletonMismatch
severity: warning
team: ops
exp_annotations:
summary: "110 正在發生的備份告警 BackupScheduleSingletonMismatch 尚未完成 receipt 閉環"
description: "BackupScheduleSingletonMismatch 正在 firing但尚無新鮮的 Telegram/AwoooP sent/received/dedup/escalation 完整 receipt未 firing 的規則不再產生假告警。"
runbook: '查 awoooi_backup_alert_receipt_stage_fresh{host="110",required_alert="BackupScheduleSingletonMismatch"} 取得缺少階段;只補 timestamp/evidence id不得寫入 token、chat raw payload 或 secret。'
- exp_labels:
alert_category: infrastructure
auto_repair: "false"
component: backup-alert-receipt
host: "110"
layer: host-backup
notification_type: TYPE-1
required_alert: BackupRetentionPolicyNotLatestOnly
severity: warning
team: ops
exp_annotations:
summary: "110 正在發生的備份告警 BackupRetentionPolicyNotLatestOnly 尚未完成 receipt 閉環"
description: "BackupRetentionPolicyNotLatestOnly 正在 firing但尚無新鮮的 Telegram/AwoooP sent/received/dedup/escalation 完整 receipt未 firing 的規則不再產生假告警。"
runbook: '查 awoooi_backup_alert_receipt_stage_fresh{host="110",required_alert="BackupRetentionPolicyNotLatestOnly"} 取得缺少階段;只補 timestamp/evidence id不得寫入 token、chat raw payload 或 secret。'