From 3bf89c32d353db9194d7db2d4d915305fd851367 Mon Sep 17 00:00:00 2001 From: Your Name Date: Fri, 3 Jul 2026 07:59:07 +0800 Subject: [PATCH] fix(backup): correct receipt aggregate promql --- docs/LOGBOOK.md | 2 +- ops/monitoring/alerts-unified.yml | 2 +- scripts/ops/backup-alert-label-contract-check.py | 2 +- scripts/ops/backup-alert-live-visibility-check.py | 2 +- .../tests/test_reboot_p0_operational_contract.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index 6089072a2..716f57117 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -53859,7 +53859,7 @@ production browser smoke: - Gitea main `0acda9f72 fix(backup): add alert receipt contract` 已經由後續 main deploy 帶入 runtime；CD `#4527` 讀回 job succeeded、deploy marker `374d4e4 -> dc4f0ec`、production deploy readback matched、success notification。 - 受控同步新版 `backup-health-textfile-exporter.py` 到 110 `/home/wooo/scripts/` 與 188 `/home/ollama/scripts/`，均先備份舊檔；立即刷新 `backup_health.prom`，110 讀回 `awoooi_backup_alert_receipt_stage_expected_info` 88 條、188 讀回 12 條。 - `scripts/ops/deploy-alerts.sh` dry-run 後正式套用：遠端規則備份、hash 驗證、Prometheus reload 與 key-rule verifier 成功；Prometheus 已載入 165 條規則。 -- runtime readback 確認 `BackupAlertReceiptMetricMissing` / `BackupAlertReceiptMetricMissing188` inactive，`BackupAlertReceiptStageMissing` rule health ok。初版逐 stage alert 會產生 100 條 pending，已立刻改成每 `host / receipt_channel` 一條聚合 alert，明細保留在 `awoooi_backup_alert_receipt_stage_fresh == 0`。 +- runtime readback 確認 `BackupAlertReceiptMetricMissing` / `BackupAlertReceiptMetricMissing188` inactive，`BackupAlertReceiptStageMissing` rule health ok。初版逐 stage alert 會產生 100 條 pending，已立刻改成每 `host / receipt_channel` 一條聚合 alert，明細保留在 `awoooi_backup_alert_receipt_stage_fresh == 0`；聚合 alert expression 必須使用 `== bool 0`，否則 PromQL 會把每個缺口 series 的值保留為 `0`，`sum(...)` 後仍為 `0`。 **驗證結果**： - `python3.11 -m pytest scripts/ops/tests/test_backup_health_textfile_exporter.py scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py ops/runner/test_cd_controlled_runtime_profile.py -q -p no:cacheprovider`：`65 passed`。 diff --git a/ops/monitoring/alerts-unified.yml b/ops/monitoring/alerts-unified.yml index e44252f88..6bf6f133e 100644 --- a/ops/monitoring/alerts-unified.yml +++ b/ops/monitoring/alerts-unified.yml @@ -1649,7 +1649,7 @@ groups: runbook: "部署新版 scripts/ops/backup-health-textfile-exporter.py 到 188，刷新 /home/ollama/node_exporter_textfiles/backup_health.prom；不得送測試 Telegram、不得讀 token、不得 restore 到 production。" - alert: BackupAlertReceiptStageMissing - expr: sum by (host, receipt_channel) (awoooi_backup_alert_receipt_stage_fresh == 0) > 0 + expr: sum by (host, receipt_channel) (awoooi_backup_alert_receipt_stage_fresh == bool 0) > 0 for: 30m labels: severity: warning diff --git a/scripts/ops/backup-alert-label-contract-check.py b/scripts/ops/backup-alert-label-contract-check.py index c74b61094..6107cbc43 100755 --- a/scripts/ops/backup-alert-label-contract-check.py +++ b/scripts/ops/backup-alert-label-contract-check.py @@ -144,7 +144,7 @@ def static_check(path: Path, baseline_path: Path) -> list[str]: rule = _require_alert(alerts, "BackupAlertReceiptStageMissing") _require_contains( str(rule.get("expr", "")), - "sum by (host, receipt_channel) (awoooi_backup_alert_receipt_stage_fresh == 0) > 0", + "sum by (host, receipt_channel) (awoooi_backup_alert_receipt_stage_fresh == bool 0) > 0", "BackupAlertReceiptStageMissing expr", ) text = _annotation_text(rule) diff --git a/scripts/ops/backup-alert-live-visibility-check.py b/scripts/ops/backup-alert-live-visibility-check.py index 512a33ec1..a35f94f25 100755 --- a/scripts/ops/backup-alert-live-visibility-check.py +++ b/scripts/ops/backup-alert-live-visibility-check.py @@ -190,7 +190,7 @@ def _escrow_required_alerts(prometheus_url: str, host: str, timeout: int) -> lis def _receipt_stage_required_alerts(prometheus_url: str, host: str, timeout: int) -> list[RequiredAlert]: expr = ( "sum by (host, receipt_channel) " - f'(awoooi_backup_alert_receipt_stage_fresh{{host="{host}"}} == 0) > 0' + f'(awoooi_backup_alert_receipt_stage_fresh{{host="{host}"}} == bool 0) > 0' ) rows = _prom_query(prometheus_url, expr, timeout) required: list[RequiredAlert] = [] diff --git a/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py b/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py index 78127cdd7..f9300dcdd 100644 --- a/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py +++ b/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py @@ -363,7 +363,7 @@ def test_backup_alerts_cover_188_gitea_mirror_subtree() -> None: assert "BackupAlertReceiptMetricMissing" in alerts assert "BackupAlertReceiptStageMissing" in alerts assert "awoooi_backup_alert_receipt_expected_info" in alerts - assert "sum by (host, receipt_channel) (awoooi_backup_alert_receipt_stage_fresh == 0) > 0" in alerts + assert "sum by (host, receipt_channel) (awoooi_backup_alert_receipt_stage_fresh == bool 0) > 0" in alerts assert "$labels.receipt_channel" in alerts assert "$value" in alerts assert "required_alert/scope/stage/required_notification_type" in alerts