diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index f1aa5e240..cd5e21edd 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -1,3 +1,23 @@ +## 2026-05-29|NoAlertsReceived2Hours 誤報與 Prometheus canonical drift 修復 + +**背景**: + +- Telegram 持續收到 `NoAlertsReceived2Hours`,來源為 `sentry` / `signoz`。 +- Live Prometheus 實際載入的 query 是 `time() - max by (source)(awoooi_alert_chain_last_success_timestamp) > 7200`,因此把 Sentry / SignOz 正常安靜時段誤判為告警鏈路故障。 +- `gitea/main` 的 `ops/monitoring/alerts-unified.yml` 已將 `NoAlertsReceived2Hours` 限縮到 `source="alertmanager"`,並新增 `SourceProviderIngestionStale` 以 24h 檢查 Sentry / SignOz provider freshness;但 110 上的 active 與 canonical 規則仍是舊版,drift guard 因 canonical 舊檔而未修復。 + +**本次調整**: + +- `scripts/ops/deploy-alerts.sh` 會同時部署 `ops/monitoring/alerts-unified.yml` 到: + - `/home/wooo/monitoring/alerts.yml` + - `/home/wooo/monitoring/alerts-unified.canonical.yml` +- deploy 後新增 canonical hash 驗證,避免 active rules 與 drift guard canonical 再次分叉。 +- 已即時執行 deploy,Prometheus reload 後: + - `NoAlertsReceived2Hours` query 已限制 `source="alertmanager"`。 + - `SourceProviderIngestionStale` query 已限制 `source=~"sentry|signoz"` 且 24h。 + - active / canonical SHA256 相同。 + - `ALERTS{alertname="NoAlertsReceived2Hours",alertstate="firing"}` 為空。 + ## 2026-05-29|IwoooS 下一步任務板補強 **背景**: diff --git a/scripts/ops/deploy-alerts.sh b/scripts/ops/deploy-alerts.sh index 8a593d46c..db226c62b 100755 --- a/scripts/ops/deploy-alerts.sh +++ b/scripts/ops/deploy-alerts.sh @@ -10,6 +10,7 @@ ALERT_RULES_FILE="ops/monitoring/alerts-unified.yml" SLO_RULES_FILE="ops/monitoring/slo-rules.yml" TARGET_HOST="192.168.0.110" TARGET_ALERTS_PATH="/home/wooo/monitoring/alerts.yml" +TARGET_ALERTS_CANONICAL_PATH="/home/wooo/monitoring/alerts-unified.canonical.yml" TARGET_SLO_PATH="/home/wooo/monitoring/slo-rules.yml" PROMETHEUS_URL="http://${TARGET_HOST}:9090" DRY_RUN="${1:-}" @@ -57,6 +58,7 @@ log "✅ YAML 語法驗證通過" # Dry run 模式 if [ "$DRY_RUN" = "--dry-run" ]; then log "DRY RUN: would deploy $ALERT_RULES_FILE to ${TARGET_HOST}:${TARGET_ALERTS_PATH}" + log "DRY RUN: would deploy $ALERT_RULES_FILE to ${TARGET_HOST}:${TARGET_ALERTS_CANONICAL_PATH}" log "DRY RUN: would deploy $SLO_RULES_FILE to ${TARGET_HOST}:${TARGET_SLO_PATH}" ALERT_COUNT=$(grep -c "alert:" "$ALERT_RULES_FILE") SLO_RECORD_COUNT=$(grep -c "record:" "$SLO_RULES_FILE") @@ -68,22 +70,29 @@ fi # 備份現有規則 ssh wooo@${TARGET_HOST} "\ cp ${TARGET_ALERTS_PATH} ${TARGET_ALERTS_PATH}.bak.\$(date +%Y%m%d%H%M%S) 2>/dev/null || true; \ + cp ${TARGET_ALERTS_CANONICAL_PATH} ${TARGET_ALERTS_CANONICAL_PATH}.bak.\$(date +%Y%m%d%H%M%S) 2>/dev/null || true; \ cp ${TARGET_SLO_PATH} ${TARGET_SLO_PATH}.bak.\$(date +%Y%m%d%H%M%S) 2>/dev/null || true" log "✅ 現有規則已備份" # 部署新規則 scp "$ALERT_RULES_FILE" wooo@${TARGET_HOST}:${TARGET_ALERTS_PATH} +scp "$ALERT_RULES_FILE" wooo@${TARGET_HOST}:${TARGET_ALERTS_CANONICAL_PATH} scp "$SLO_RULES_FILE" wooo@${TARGET_HOST}:${TARGET_SLO_PATH} log "✅ 規則已複製到 ${TARGET_HOST}" LOCAL_ALERTS_SHA="$(file_sha "$ALERT_RULES_FILE")" REMOTE_ALERTS_SHA="$(remote_file_sha "$TARGET_ALERTS_PATH")" +REMOTE_ALERTS_CANONICAL_SHA="$(remote_file_sha "$TARGET_ALERTS_CANONICAL_PATH")" LOCAL_SLO_SHA="$(file_sha "$SLO_RULES_FILE")" REMOTE_SLO_SHA="$(remote_file_sha "$TARGET_SLO_PATH")" if [ "$LOCAL_ALERTS_SHA" != "$REMOTE_ALERTS_SHA" ]; then echo "ERROR: 遠端 alerts.yml hash 不一致 local=${LOCAL_ALERTS_SHA} remote=${REMOTE_ALERTS_SHA}" exit 1 fi +if [ "$LOCAL_ALERTS_SHA" != "$REMOTE_ALERTS_CANONICAL_SHA" ]; then + echo "ERROR: 遠端 alerts-unified.canonical.yml hash 不一致 local=${LOCAL_ALERTS_SHA} remote=${REMOTE_ALERTS_CANONICAL_SHA}" + exit 1 +fi if [ "$LOCAL_SLO_SHA" != "$REMOTE_SLO_SHA" ]; then echo "ERROR: 遠端 slo-rules.yml hash 不一致 local=${LOCAL_SLO_SHA} remote=${REMOTE_SLO_SHA}" exit 1