fix(recovery): detect all-host reboot recovery gaps

2026-06-30 18:21:56 +08:00
parent 66ec3c92e5
commit c6ec7a3b71
14 changed files with 1133 additions and 13 deletions
--- a/ops/maintenance/maintenance.html
+++ b/ops/maintenance/maintenance.html
@@ -0,0 +1,78 @@
+<!doctype html>
+<html lang="zh-Hant">
+  <head>
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <meta name="robots" content="noindex">
+    <title>系統維護中</title>
+    <style>
+      :root {
+        color-scheme: light;
+        --ink: #172026;
+        --muted: #52606d;
+        --line: #d8dee5;
+        --accent: #0f766e;
+        --bg: #f7f8f8;
+      }
+      * {
+        box-sizing: border-box;
+      }
+      body {
+        margin: 0;
+        min-height: 100vh;
+        display: grid;
+        place-items: center;
+        background: var(--bg);
+        color: var(--ink);
+        font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", "Noto Sans TC", sans-serif;
+      }
+      main {
+        width: min(92vw, 640px);
+        padding: 32px 0;
+      }
+      .status {
+        display: inline-flex;
+        align-items: center;
+        gap: 10px;
+        margin-bottom: 20px;
+        color: var(--accent);
+        font-size: 15px;
+        font-weight: 700;
+      }
+      .dot {
+        width: 10px;
+        height: 10px;
+        border-radius: 999px;
+        background: var(--accent);
+      }
+      h1 {
+        margin: 0 0 16px;
+        font-size: clamp(32px, 8vw, 56px);
+        line-height: 1.05;
+        letter-spacing: 0;
+      }
+      p {
+        margin: 0 0 16px;
+        color: var(--muted);
+        font-size: 18px;
+        line-height: 1.7;
+      }
+      .meta {
+        margin-top: 28px;
+        padding-top: 18px;
+        border-top: 1px solid var(--line);
+        font-size: 14px;
+        color: var(--muted);
+      }
+    </style>
+  </head>
+  <body>
+    <main>
+      <div class="status"><span class="dot"></span>AWOOOI 維護模式</div>
+      <h1>系統正在恢復服務</h1>
+      <p>我們偵測到主機或上游服務正在重啟，流量暫時切到維護頁。這個頁面不依賴內部資料庫或應用程式，因此在主要服務恢復前仍可正常顯示。</p>
+      <p>系統會持續自動檢查，服務恢復後會切回正式網站。</p>
+      <div class="meta">HTTP 502 / 503 / 504 fallback page. No application secrets, cookies, or user data are loaded here.</div>
+    </main>
+  </body>
+</html>
--- a/ops/maintenance/nginx-502-maintenance-snippet.conf
+++ b/ops/maintenance/nginx-502-maintenance-snippet.conf
@@ -0,0 +1,16 @@
+# Include this inside each public server {} that proxies to internal AWOOOI,
+# StockPlatform, MOMO, Tsenyang, Bitan, Gitea, Harbor, or registry upstreams.
+#
+# The static file must be deployed outside the application containers, for
+# example /var/www/maintenance/maintenance.html on the edge Nginx host.
+
+proxy_intercept_errors on;
+error_page 502 503 504 /__awoooi-maintenance.html;
+
+location = /__awoooi-maintenance.html {
+    internal;
+    root /var/www/maintenance;
+    try_files /maintenance.html =503;
+    add_header Cache-Control "no-store, no-cache, must-revalidate, max-age=0" always;
+    add_header X-AWOOOI-Fallback "local-maintenance" always;
+}
--- a/ops/monitoring/alerts-unified.yml
+++ b/ops/monitoring/alerts-unified.yml
@@ -60,10 +60,12 @@ groups:
          severity: critical
          layer: systemd-188
          team: ops
+          alert_category: reboot_recovery
+          notification_type: TYPE-3
          auto_repair: "false"
        annotations:
          summary: "主機 {{ $labels.host }} 不可達"
-          description: "Node Exporter 無回應超過 1 分鐘"
+          description: "Node Exporter 無回應超過 1 分鐘；若發生在全主機重啟或 99/VMware restart，必須立刻送 Telegram 並啟動 reboot recovery SLO scorecard。"

      - alert: HostHighCpuLoad
        # 2026-05-05 ogt + Codex: keep this as early warning only.
@@ -133,6 +135,97 @@ groups:
          description: "磁碟使用率超過 85%"
          auto_repair_action: "ssh {{ $labels.instance }} 'echo \"=== CPU TOP ===\"; ps aux --sort=-%cpu | head -15; echo \"=== MEMORY ===\"; free -h; echo \"=== DISK ===\"; df -h; echo \"=== LOAD ===\"; uptime'"

+  # =========================================================================
+  # Reboot event / 10-minute recovery SLO alerts
+  # =========================================================================
+  - name: reboot_recovery_slo_alerts
+    interval: 1m
+    rules:
+      - alert: RebootEventDetectorMissing
+        expr: absent(awoooi_reboot_event_detected{scope="99_110_111_112_120_121_188"})
+        for: 15m
+        labels:
+          severity: warning
+          layer: systemd-110
+          component: reboot-event-detector
+          team: ops
+          alert_category: reboot_recovery
+          notification_type: TYPE-1
+          auto_repair: "false"
+        annotations:
+          summary: "全主機重啟事件偵測器沒有輸出"
+          description: "reboot-auto-recovery SLO exporter 沒有輸出 awoooi_reboot_event_detected；99/110/111/112/120/121/188 重啟後無法自動判斷事件開始時間。"
+          runbook: "檢查 110 awoooi-reboot-auto-recovery-slo.timer / service、host-probe、reboot-event-state.json 與 node-exporter textfile collector。"
+
+      - alert: HostRebootEventDetected
+        expr: awoooi_reboot_event_host_rebooted{host=~"99|110|111|112|120|121|188"} == 1
+        for: 1m
+        labels:
+          severity: warning
+          layer: systemd-110
+          component: reboot-event-detector
+          team: ops
+          alert_category: reboot_recovery
+          notification_type: TYPE-1
+          auto_repair: "false"
+        annotations:
+          summary: "偵測到主機 {{ $labels.host }} 已重啟"
+          description: "boot_id 或 fresh uptime 顯示 {{ $labels.host }} 位於 10 分鐘重啟恢復視窗內；SLO scorecard 會固定追蹤 remaining seconds、public route、backup 與 freshness。"
+          runbook: "讀取 reboot_auto_recovery_slo.prom 與最新 scorecard.json；不要手動重啟服務，先依 fixed_triage_order 查 99 VM、host boot、public route、K3s/registry、Stock freshness、backup、Telegram delivery。"
+
+      - alert: RebootEventRequiredHostMissing
+        expr: awoooi_reboot_event_required_host_observed{host=~"99|110|111|112|120|121|188"} == 0
+        for: 2m
+        labels:
+          severity: critical
+          layer: systemd-110
+          component: reboot-event-detector
+          team: ops
+          alert_category: reboot_recovery
+          notification_type: TYPE-3
+          auto_repair: "false"
+        annotations:
+          summary: "重啟偵測缺少必要主機 {{ $labels.host }}"
+          description: "{{ $labels.host }} 沒有出現在 host boot probe；10 分鐘自動恢復不能宣稱成立。"
+          runbook: "先確認 99 VMware / VM power 狀態與 node-exporter/Windows exporter，再重跑 reboot-auto-recovery-host-probe.sh；不得用刪除 required host 的方式消紅。"
+
+      - alert: RebootAutoRecoverySLOMissed
+        expr: |
+          (awoooi_reboot_event_detected{scope="99_110_111_112_120_121_188"} == 1)
+          and on(scope)
+          (awoooi_reboot_event_target_seconds_remaining{scope="99_110_111_112_120_121_188"} <= 0)
+          and on(scope)
+          (awoooi_reboot_auto_recovery_slo_ready{scope="99_110_111_112_120_121_188"} == 0)
+        for: 1m
+        labels:
+          severity: critical
+          layer: systemd-110
+          component: reboot-auto-recovery-slo
+          team: ops
+          alert_category: reboot_recovery
+          notification_type: TYPE-3
+          auto_repair: "false"
+        annotations:
+          summary: "全主機重啟後 10 分鐘恢復 SLO 未達成"
+          description: "偵測到重啟事件，但 10 分鐘期限已過且 awoooi_reboot_auto_recovery_slo_ready 仍為 0。"
+          runbook: "直接讀最新 scorecard.json 的 active_blockers 與 fixed_triage_order；先處理 99/VMware/VM power、public maintenance fallback、registry/K3s、Stock freshness、backup health 與 Telegram delivery。"
+
+      - alert: RebootAutoRecoverySLOExporterStale
+        expr: time() - awoooi_reboot_auto_recovery_slo_last_run_timestamp{scope="99_110_111_112_120_121_188"} > 600
+        for: 10m
+        labels:
+          severity: warning
+          layer: systemd-110
+          component: reboot-auto-recovery-slo
+          team: ops
+          alert_category: reboot_recovery
+          notification_type: TYPE-1
+          auto_repair: "false"
+        annotations:
+          summary: "重啟恢復 SLO exporter 超過 10 分鐘未更新"
+          description: "reboot auto-recovery SLO textfile stale；即使服務看似正常，也不能證明 10 分鐘內恢復。"
+          runbook: "檢查 systemd timer、/home/wooo/reboot-recovery artifacts、node-exporter textfile collector。"
+
  # =========================================================================
  # Host runaway process / CI load classification
  # =========================================================================
@@ -996,6 +1089,26 @@ groups:
          summary: "外部網站 bitan.wooo.work 離線"
          description: "bitan.wooo.work 探測失敗超過 3 分鐘，容器 bitan-app (188) 可能需要重啟"

+      - alert: PublicRouteServing5xxWithoutMaintenanceFallback
+        expr: |
+          probe_http_status_code{
+            job="blackbox-http",
+            instance=~"https?://(awoooi\\.wooo\\.work|stock\\.wooo\\.work|mo\\.wooo\\.work|bitan\\.wooo\\.work|www\\.tsenyang\\.com).*"
+          } >= 500
+        for: 2m
+        labels:
+          severity: critical
+          layer: external
+          component: public-maintenance-fallback
+          team: ops
+          alert_category: reboot_recovery
+          notification_type: TYPE-3
+          auto_repair: "false"
+        annotations:
+          summary: "Public route 正在對使用者回 {{ $value }}：{{ $labels.instance }}"
+          description: "重啟或 upstream failure 期間 public route 仍露出 5xx；應由 L0 Nginx maintenance fallback 或 L1 external cloud/CDN 維護頁接手。"
+          runbook: "依 docs/runbooks/PUBLIC-MAINTENANCE-FALLBACK-RUNBOOK.md 部署 ops/maintenance/maintenance.html 與 Nginx 502/503/504 intercept；整段 edge unreachable 時啟用外部 CDN/static fallback。"
+
      - alert: ExternalSiteSSLExpiringSoon
        expr: probe_ssl_earliest_cert_expiry{job="blackbox-http"} - time() < 14 * 24 * 3600
        for: 1h
@@ -1357,6 +1470,40 @@ groups:
          description: "{{ $labels.exported_job }} 最新成功證據超過 {{ $labels.max_age_hours }} 小時或不存在；來源 {{ $labels.source }}，目標 {{ $labels.target }}。"
          runbook: "先檢查備份 log 與磁碟空間，再手動執行對應備份；禁止直接刪除舊備份或 production 資料"

+      - alert: BackupCoverageDomainMetricMissing
+        expr: absent(awoooi_backup_coverage_domain_expected_info{host="110"})
+        for: 20m
+        labels:
+          severity: warning
+          layer: host-backup
+          component: backup-coverage
+          host: "110"
+          team: ops
+          alert_category: infrastructure
+          notification_type: TYPE-1
+          auto_repair: "false"
+        annotations:
+          summary: "110 備份覆蓋 domain 指標缺失"
+          description: "backup-health exporter 沒有輸出 host/database/website/service/package/tool/log 聚合覆蓋指標，無法快速判斷完整備份是否還在運作。"
+          runbook: "部署新版 scripts/ops/backup-health-textfile-exporter.py，刷新 /home/wooo/node_exporter_textfiles/backup_health.prom。"
+
+      - alert: BackupCoverageDomainStale
+        expr: awoooi_backup_coverage_domain_fresh{host="110"} == 0
+        for: 15m
+        labels:
+          severity: critical
+          layer: host-backup
+          component: backup-coverage
+          host: "110"
+          team: ops
+          alert_category: infrastructure
+          notification_type: TYPE-3
+          auto_repair: "false"
+        annotations:
+          summary: "110 備份覆蓋 domain 過舊：{{ $labels.domain }}"
+          description: "{{ $labels.domain }} 需要的備份證據 {{ $labels.required_jobs }} 未全部新鮮；重啟恢復不能宣稱備份完整。"
+          runbook: "先跑 `scripts/backup/backup-status.sh --no-notify` 和 backup-health exporter readback，修復對應 required_jobs；禁止刪除、prune、restore 或覆蓋 production。"
+
      - alert: BackupAggregateRunFailed
        expr: awoooi_backup_last_run_failed_count{host="110",exported_job="backup_all"} > 0
        for: 10m