From 63003f7330215dbfb580422360705a4de4322cf5 Mon Sep 17 00:00:00 2001 From: Your Name Date: Fri, 3 Jul 2026 07:55:55 +0800 Subject: [PATCH] fix(backup): aggregate receipt gap alerts --- docs/LOGBOOK.md | 21 +++++++++++ docs/runbooks/FULL-STACK-COLD-START-SOP.md | 1 + docs/runbooks/REBOOT-RECOVERY-SOP.md | 2 +- ops/monitoring/alerts-unified.yml | 8 ++--- .../ops/backup-alert-label-contract-check.py | 11 +++--- .../ops/backup-alert-live-visibility-check.py | 35 +++++-------------- .../test_reboot_p0_operational_contract.py | 7 ++-- 7 files changed, 44 insertions(+), 41 deletions(-) diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index c0a175406..6089072a2 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -53852,3 +53852,24 @@ production browser smoke: **下一步**： - 整合 Gitea main 最新 commit 後 push；CD 完成後讀回 Prometheus / Alertmanager / production scorecard，確認新 receipt metrics / rules 進 runtime，再往下一個 P0 blocker（99 VMware autostart / full Gitea dump / DB/settings/issues/packages/LFS / service green）推進。 + +## 2026-07-03 — P0-5 / P0-6 backup receipt runtime apply 與噪音收斂 + +**完成內容**： +- Gitea main `0acda9f72 fix(backup): add alert receipt contract` 已經由後續 main deploy 帶入 runtime；CD `#4527` 讀回 job succeeded、deploy marker `374d4e4 -> dc4f0ec`、production deploy readback matched、success notification。 +- 受控同步新版 `backup-health-textfile-exporter.py` 到 110 `/home/wooo/scripts/` 與 188 `/home/ollama/scripts/`，均先備份舊檔；立即刷新 `backup_health.prom`，110 讀回 `awoooi_backup_alert_receipt_stage_expected_info` 88 條、188 讀回 12 條。 +- `scripts/ops/deploy-alerts.sh` dry-run 後正式套用：遠端規則備份、hash 驗證、Prometheus reload 與 key-rule verifier 成功；Prometheus 已載入 165 條規則。 +- runtime readback 確認 `BackupAlertReceiptMetricMissing` / `BackupAlertReceiptMetricMissing188` inactive，`BackupAlertReceiptStageMissing` rule health ok。初版逐 stage alert 會產生 100 條 pending，已立刻改成每 `host / receipt_channel` 一條聚合 alert，明細保留在 `awoooi_backup_alert_receipt_stage_fresh == 0`。 + +**驗證結果**： +- `python3.11 -m pytest scripts/ops/tests/test_backup_health_textfile_exporter.py scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py ops/runner/test_cd_controlled_runtime_profile.py -q -p no:cacheprovider`：`65 passed`。 +- `python3.11 scripts/ops/backup-alert-label-contract-check.py`：`BACKUP_ALERT_LABEL_CONTRACT_OK`。 +- py_compile、Prometheus / baseline YAML parse、`guard-gitea-runner-pressure.py --root .`、`git diff --check`：通過。 + +**仍維持**： +- 沒有讀 secret / token / `.env` / raw sessions / SQLite / auth；沒有讀 Telegram token、runner token 或 chat raw payload。 +- 沒有使用 GitHub / gh / GitHub API / GitHub Actions。 +- 沒有重啟主機，沒有 Docker / Nginx / K3s / DB / firewall restart，沒有 workflow_dispatch，沒有 DROP / TRUNCATE / restore / prune，沒有送 Telegram 測試訊息。 + +**下一步**： +- commit / push 聚合告警修正；再套用新版 Prometheus rule，確認 `BackupAlertReceiptStageMissing` pending/firing 數量聚合成 110 / 188 host-level summary，而不是逐 stage 噪音。 diff --git a/docs/runbooks/FULL-STACK-COLD-START-SOP.md b/docs/runbooks/FULL-STACK-COLD-START-SOP.md index 485b3c70f..2457450a2 100644 --- a/docs/runbooks/FULL-STACK-COLD-START-SOP.md +++ b/docs/runbooks/FULL-STACK-COLD-START-SOP.md @@ -2604,6 +2604,7 @@ Prometheus 必須載入： 判讀規則： - `awoooi_backup_alert_receipt_stage_fresh == 0` 代表該 backup alert 的 `sent / received / dedup / escalation` 某段沒有新鮮證據；不能宣稱 Telegram/AwoooP receipt 完成。 +- `BackupAlertReceiptStageMissing` 必須聚合成每個 `host / receipt_channel` 一條 alert；明細用 `awoooi_backup_alert_receipt_stage_fresh{host="...",receipt_channel="..."} == 0` 查 `required_alert / scope / stage / required_notification_type`，避免每個 stage 都打 Telegram 造成噪音。 - `required_alert` 是原始 backup alert 名稱；不得用 Prometheus 保留 label `alertname` 承載原始 alert，避免被 meta alert 覆蓋。 - `required_notification_type` 是原始 backup alert 應走的類型；meta alert 自己的 `notification_type` 可維持 `TYPE-1`。 - `scripts/ops/backup-alert-label-contract-check.py` 是 source rule contract；`scripts/ops/backup-alert-live-visibility-check.py` 是 Prometheus / Alertmanager live visibility contract。 diff --git a/docs/runbooks/REBOOT-RECOVERY-SOP.md b/docs/runbooks/REBOOT-RECOVERY-SOP.md index 685d35d71..3a4ffde60 100644 --- a/docs/runbooks/REBOOT-RECOVERY-SOP.md +++ b/docs/runbooks/REBOOT-RECOVERY-SOP.md @@ -413,7 +413,7 @@ scripts/ops/backup-alert-live-visibility-check.py --prometheus-url http://192.16 判讀規則： - `required_alert` 是原始 backup alert，例如 `BackupJobStale`、`BackupCoverageDomainStale`、`GiteaPrivateBundleBackupIncomplete`；不得用 Prometheus 保留 label `alertname` 承載它。 -- 每個 required alert 都要有 `sent / received / dedup / escalation` 四段 marker；缺一段就是 `BackupAlertReceiptStageMissing`。 +- 每個 required alert 都要有 `sent / received / dedup / escalation` 四段 marker；缺段明細由 `awoooi_backup_alert_receipt_stage_fresh == 0` 保留，`BackupAlertReceiptStageMissing` 只允許聚合成每個 `host / receipt_channel` 一條，避免 Telegram/AwoooP 噪音淹沒真問題。 - marker 只能是 `/backup/alert-receipts/*.last_success` 這類 timestamp / evidence id；不得寫 Telegram token、raw payload、chat id 原文或 secret。 - 這個 receipt proof 不取代真正 backup / restore proof；Gitea bundle、Gitea full dump、DB/settings/issues/packages/LFS、offsite、restore drill 仍要各自綠。 diff --git a/ops/monitoring/alerts-unified.yml b/ops/monitoring/alerts-unified.yml index 071d2f683..e44252f88 100644 --- a/ops/monitoring/alerts-unified.yml +++ b/ops/monitoring/alerts-unified.yml @@ -1649,7 +1649,7 @@ groups: runbook: "部署新版 scripts/ops/backup-health-textfile-exporter.py 到 188，刷新 /home/ollama/node_exporter_textfiles/backup_health.prom；不得送測試 Telegram、不得讀 token、不得 restore 到 production。" - alert: BackupAlertReceiptStageMissing - expr: awoooi_backup_alert_receipt_stage_fresh == 0 + expr: sum by (host, receipt_channel) (awoooi_backup_alert_receipt_stage_fresh == 0) > 0 for: 30m labels: severity: warning @@ -1660,9 +1660,9 @@ groups: notification_type: TYPE-1 auto_repair: "false" annotations: - summary: "備份告警 {{ $labels.required_alert }} 缺少 {{ $labels.stage }} receipt" - description: "{{ $labels.required_alert }} / scope={{ $labels.scope }} 尚無新鮮 {{ $labels.stage }} receipt marker；required_notification_type={{ $labels.required_notification_type }}，channel={{ $labels.receipt_channel }}，max_age_hours={{ $labels.max_age_hours }}。這代表 Telegram/AwoooP sent/received/dedup/escalation 鏈尚未被證明，不能宣稱備份告警閉環。" - runbook: "先讀 /backup/alert-receipts/{{ $labels.required_alert }}.*.last_success 或 AwoooP outbound receipt readback，標準 receipt_channel 為 telegram_awooop；補脫敏 evidence marker 時只允許 timestamp/evidence id，不得寫入 token、chat raw payload 或 secret。" + summary: "{{ $labels.host }} 備份告警 receipt 缺少階段證據" + description: "{{ $labels.host }} / channel={{ $labels.receipt_channel }} 目前有 {{ $value }} 個 backup alert receipt stage 缺少新鮮 marker；這代表 Telegram/AwoooP sent/received/dedup/escalation 鏈尚未被證明，不能宣稱備份告警閉環。" + runbook: "先查 `awoooi_backup_alert_receipt_stage_fresh{host=\"{{ $labels.host }}\",receipt_channel=\"{{ $labels.receipt_channel }}\"} == 0` 取得 required_alert/scope/stage/required_notification_type 明細；標準 receipt_channel 為 telegram_awooop，再讀 /backup/alert-receipts/*.last_success 或 AwoooP outbound receipt readback；補脫敏 evidence marker 時只允許 timestamp/evidence id，不得寫入 token、chat raw payload 或 secret。" - alert: BackupAggregateRunFailed expr: sum by (host) (awoooi_backup_last_run_failed_count{host="110",exported_job!="backup_all"}) > 0 diff --git a/scripts/ops/backup-alert-label-contract-check.py b/scripts/ops/backup-alert-label-contract-check.py index 8f08c6ebb..c74b61094 100755 --- a/scripts/ops/backup-alert-label-contract-check.py +++ b/scripts/ops/backup-alert-label-contract-check.py @@ -144,19 +144,18 @@ def static_check(path: Path, baseline_path: Path) -> list[str]: rule = _require_alert(alerts, "BackupAlertReceiptStageMissing") _require_contains( str(rule.get("expr", "")), - "awoooi_backup_alert_receipt_stage_fresh == 0", + "sum by (host, receipt_channel) (awoooi_backup_alert_receipt_stage_fresh == 0) > 0", "BackupAlertReceiptStageMissing expr", ) text = _annotation_text(rule) for required_label in [ - "$labels.required_alert", - "$labels.scope", - "$labels.stage", - "$labels.required_notification_type", + "$labels.host", "$labels.receipt_channel", + "$value", + "required_alert/scope/stage/required_notification_type", ]: _require_contains(text, required_label, "BackupAlertReceiptStageMissing annotations") - lines.append("OK BackupAlertReceiptStageMissing uses required_alert/scope/stage/receipt labels") + lines.append("OK BackupAlertReceiptStageMissing aggregates by host/receipt channel and points to detail labels") return lines diff --git a/scripts/ops/backup-alert-live-visibility-check.py b/scripts/ops/backup-alert-live-visibility-check.py index de3d3d782..512a33ec1 100755 --- a/scripts/ops/backup-alert-live-visibility-check.py +++ b/scripts/ops/backup-alert-live-visibility-check.py @@ -188,24 +188,19 @@ def _escrow_required_alerts(prometheus_url: str, host: str, timeout: int) -> lis def _receipt_stage_required_alerts(prometheus_url: str, host: str, timeout: int) -> list[RequiredAlert]: - expr = f'awoooi_backup_alert_receipt_stage_fresh{{host="{host}"}} == 0' + expr = ( + "sum by (host, receipt_channel) " + f'(awoooi_backup_alert_receipt_stage_fresh{{host="{host}"}} == 0) > 0' + ) rows = _prom_query(prometheus_url, expr, timeout) required: list[RequiredAlert] = [] for row in rows: labels = _metric_labels(row) - required_alert = labels.get("required_alert") - scope = labels.get("scope") - stage = labels.get("stage") receipt_channel = labels.get("receipt_channel") - required_notification_type = labels.get("required_notification_type") missing = [ label for label, value in { - "required_alert": required_alert, - "scope": scope, - "stage": stage, "receipt_channel": receipt_channel, - "required_notification_type": required_notification_type, }.items() if not value ] @@ -217,22 +212,11 @@ def _receipt_stage_required_alerts(prometheus_url: str, host: str, timeout: int) { **COMMON_LABELS, "host": host, - "required_alert": str(required_alert), - "scope": str(scope), - "stage": str(stage), "receipt_channel": str(receipt_channel), - "required_notification_type": str(required_notification_type), }, ) ) - return sorted( - required, - key=lambda alert: ( - alert.labels["required_alert"], - alert.labels["scope"], - alert.labels["stage"], - ), - ) + return sorted(required, key=lambda alert: alert.labels["receipt_channel"]) def live_check(prometheus_url: str, alertmanager_url: str, host: str, timeout: int) -> list[str]: @@ -256,13 +240,10 @@ def live_check(prometheus_url: str, alertmanager_url: str, host: str, timeout: i receipt_stage_alerts = _receipt_stage_required_alerts(prometheus_url, host, timeout) required_alerts.extend(receipt_stage_alerts) if receipt_stage_alerts: - samples = ", ".join( - f"{alert.labels['required_alert']}:{alert.labels['stage']}" - for alert in receipt_stage_alerts[:8] - ) - suffix = "" if len(receipt_stage_alerts) <= 8 else f", ... +{len(receipt_stage_alerts) - 8}" lines.append( - f"OK backup alert receipt gap metrics require {len(receipt_stage_alerts)} alert(s): {samples}{suffix}" + "OK backup alert receipt gap metrics require " + f"{len(receipt_stage_alerts)} aggregated alert(s): " + + ", ".join(alert.labels["receipt_channel"] for alert in receipt_stage_alerts) ) else: lines.append("OK backup alert receipt markers are fresh; no receipt stage alert required") diff --git a/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py b/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py index 00f15058a..78127cdd7 100644 --- a/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py +++ b/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py @@ -363,9 +363,10 @@ def test_backup_alerts_cover_188_gitea_mirror_subtree() -> None: assert "BackupAlertReceiptMetricMissing" in alerts assert "BackupAlertReceiptStageMissing" in alerts assert "awoooi_backup_alert_receipt_expected_info" in alerts - assert "awoooi_backup_alert_receipt_stage_fresh == 0" in alerts - assert "$labels.required_alert" in alerts - assert "$labels.required_notification_type" in alerts + assert "sum by (host, receipt_channel) (awoooi_backup_alert_receipt_stage_fresh == 0) > 0" in alerts + assert "$labels.receipt_channel" in alerts + assert "$value" in alerts + assert "required_alert/scope/stage/required_notification_type" in alerts assert "telegram_awooop" in alerts