From 068c18e2f00d69deaba1e68e45ed0470839f0a45 Mon Sep 17 00:00:00 2001 From: Your Name Date: Thu, 2 Jul 2026 22:24:51 +0800 Subject: [PATCH] fix(reboot): expose active blocker alert metrics --- docs/LOGBOOK.md | 16 ++++++++ docs/runbooks/FULL-STACK-COLD-START-SOP.md | 4 +- ...r-inserted-requirements-priority-ledger.md | 16 ++++---- ops/monitoring/alerts-unified.yml | 35 +++++++++++++++++ .../test_host_pressure_alert_contract.py | 18 +++++++++ .../reboot-auto-recovery-slo-exporter.sh | 39 +++++++++++++++++++ .../reboot-recovery-readiness-audit.sh | 4 ++ ...test_reboot_auto_recovery_slo_installer.py | 11 ++++++ .../test_reboot_p0_operational_contract.py | 5 +++ 9 files changed, 140 insertions(+), 8 deletions(-) diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index 37d1ba0bf..d02683967 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -1,3 +1,19 @@ +## 2026-07-02 — 22:22 P0-006 reboot SLO per-blocker 告警投影 + +**完成內容**： +- `reboot-auto-recovery-slo-exporter.sh` 已將同一份 `scorecard.json` 的 `active_blockers` 與 `primary_blocker` 投影成 node-exporter textfile metrics：`awoooi_reboot_auto_recovery_slo_active_blocker{blocker=...}`、`awoooi_reboot_auto_recovery_slo_primary_blocker{blocker=...}`。 +- `ops/monitoring/alerts-unified.yml` 新增 `RebootAutoRecoveryActiveBlocker`，讓 Telegram / Alertmanager 不再只看到 `RebootAutoRecoverySLOMissed` 總告警，而是能直接看到 `windows99_remote_execution_channel_unavailable`、`backup_core_green_not_1`、`wazuh_dashboard_degraded`、`host_unreachable_after_reboot` 等具名 blocker。 +- 同步新增 `RebootAutoRecoveryActiveBlockerMetricMissing`，當 blocker_count > 0 但 per-blocker metric 缺失時，直接暴露監控鏈缺口。 +- `FULL-STACK-COLD-START-SOP.md` 升到 v1.96，固定規則：重啟後先看 per-blocker metric，再按 Windows99 / host probe / backup / Wazuh / public fallback lane 處理，不得重新臨場發明排查順序。 + +**驗證**： +- `bash -n scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh scripts/reboot-recovery/reboot-recovery-readiness-audit.sh`：通過。 +- `python3.11` YAML readback：`ops/monitoring/alerts-unified.yml groups=23`、`ops/monitoring/slo-rules.yml groups=1`。 +- `DATABASE_URL=postgresql+asyncpg://test:test@localhost/test PYTHONPATH=apps/api python3.11 -m pytest scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_installer.py scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py scripts/ops/tests/test_host_pressure_alert_contract.py -q -p no:cacheprovider`：`17 passed`。 + +**仍維持**： +- 這是 source / monitoring rule 修正，未讀 secret / token / `.env` / raw sessions / SQLite / auth；未使用 GitHub / gh；未 workflow_dispatch；未重啟主機 / VM / Docker / Nginx / K3s / DB / firewall；未 DROP / TRUNCATE / restore / prune。 + ## 2026-07-02 — 22:02 P0-006 public maintenance runtime readback closure **完成內容**： diff --git a/docs/runbooks/FULL-STACK-COLD-START-SOP.md b/docs/runbooks/FULL-STACK-COLD-START-SOP.md index a3d3780fb..7578b0950 100644 --- a/docs/runbooks/FULL-STACK-COLD-START-SOP.md +++ b/docs/runbooks/FULL-STACK-COLD-START-SOP.md @@ -1,6 +1,6 @@ # AWOOOI 全棧冷啟動與主機重啟 SOP -> Version: v1.95 +> Version: v1.96 > Last updated: 2026-07-02 Asia/Taipei > Scope: 110 / 120 / 121 / 188 full-stack reboot recovery. 112 Kali is recorded as P3 optional and is not part of this recovery path. @@ -24,6 +24,8 @@ v1.94 Windows 99 / VMware verifier rule：99 主機與 guest VM 自動啟動不 v1.95 Windows 99 management-channel readback rule：若 99 可 ping / RDP / TCP，但 `windows99-vmware-autostart.ps1 -Mode Verify` 尚未收回，不得只寫成「等 verifier」。必須先跑 `scripts/reboot-recovery/windows99-management-channel-probe.py --output `，並把結果用 `reboot-auto-recovery-slo-scorecard.py --windows99-management-file ` 接入同一份 SLO scorecard。此 probe 只做 no-secret readback：TCP `22 / 135 / 445 / 2179 / 3389 / 5985 / 5986`、SSH BatchMode publickey、WinRM port、RDP console reachability 與 VMConnect / local console reachability；不得讀 Windows 密碼、不得啟動 VM、不得重啟、不得改 Windows Update。2026-07-02 18:28 live readback：`host_reachable=true`、`rdp_console_reachable=true`、`hyperv_vmconnect_open=true`、`local_console_channel_reachable=true`、`console_collection_channels=["rdp_console","hyperv_vmconnect"]`、`ssh_batch.status=permission_denied`、`winrm_http_open=false`、`winrm_https_open=false`、`remote_execution_channel_ready=false`；因此 active blockers 必須包含 `windows99_remote_execution_channel_unavailable` 與 `windows99_vmware_autostart_readback_missing`，next action 固定為 `restore_windows99_no_secret_management_channel_or_collect_local_console_verify_readback_then_rerun_reboot_scorecard_no_reboot`。 +v1.96 reboot SLO per-blocker alert projection rule：重啟後不得只看 `awoooi_reboot_auto_recovery_slo_blocker_count` 或 `RebootAutoRecoverySLOMissed` 總告警。`scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh` 必須從同一份 `scorecard.json` 產生 `awoooi_reboot_auto_recovery_slo_active_blocker{blocker=...}` 與 `awoooi_reboot_auto_recovery_slo_primary_blocker{blocker=...}` textfile metrics；Prometheus 必須載入 `RebootAutoRecoveryActiveBlocker` 與 `RebootAutoRecoveryActiveBlockerMetricMissing`。Telegram / Alertmanager 看到 blocker label 後，先按 `blocker` 指向的固定 lane 處理：`windows99_*` 走 no-secret collector 或 console verify、`backup_*` 走 backup-status / exporter readback、`wazuh_dashboard_degraded` 走 Wazuh registry/API 只讀、`host_*` / `reboot_event_*` 走 host-probe / reboot-event-state，同時禁止 reboot、Docker / systemd / Nginx / DB / firewall restart、restore、prune、delete。若 blocker_count 大於 0 但 active_blocker metric 缺失，視為監控鏈缺口，不可宣稱「有告警」已足夠。 + 2026-07-02 110 control-path / Harbor recovery receipt rule：若 Gitea Harbor repair queue 仍保留 `harbor_110_remote_ssh_publickey_auth_stalled`、remote-control unavailable、jobs stale 或 historical failure，但同一輪本地證據同時證明 `wooo` command path ready、110 local Harbor `/v2/` ready、public/internal registry `/v2/` 回 `401`，則該 Gitea Harbor repair 失敗只能列為 historical queue metadata，不得再當成 current SSH blocker。必須用 `/api/v1/agents/harbor-registry-controlled-recovery-receipt` 或同等 validator 合併 `diagnose-110-ssh-publickey-auth.sh`、`recover-110-control-path-and-harbor-local.sh --check`、public Gitea queue readback 與 registry `/v2/` verifier，並把機器可讀結果寫入 `docs/operations/harbor-110-control-path-recovery-readback-2026-07-02.snapshot.json` 類型的 snapshot。2026-07-02 live receipt 顯示：public/internal registry `/v2/` 均為 `401`、latest visible CD `#4335` 為 `Success`、Gitea Harbor repair failure 已是 `historical_after_latest_cd_success=true`；active blockers 收斂為 110 controlled CD lane config / binary / registration / service guardrail、active action container pressure，以及 Gitea CD jobs head-SHA / stale readback mismatch。若 local-console output 只有 `AWOOOI_110_CONTROLLED_CD_LANE_READY` marker，non110 runner parser 不得從 110 `BLOCKER` 行推導 non110 blocker；non110 只有看到 `AWOOOI_NON110_RUNNER_READY` marker 才能列入 active blocker。 2026-07-02 110 controlled CD lane fail-closed enforcer staging rule：110 runner 壓力事故後，legacy / generic runner 仍必須 fail-closed；但 `awoooi-cd-lane-drain.service` 的非 secret staging artifact 不得再被 enforcer 無差別封回 stub。`scripts/reboot-recovery/enforce-110-runner-failclosed.sh` 只有在 `config.yaml` 符合 `capacity <= 1`、只含 `awoooi-host:host` 與 `awoooi-ubuntu:docker://192.168.0.110:5000/awoooi/ci-runner:act-22.04`、binary 是 executable ELF、systemd unit 具備 `ConditionPathExists=/home/wooo/awoooi-cd-lane-drain/data/.runner`、`CPUAccounting` / `MemoryAccounting` / `TasksAccounting` / `NoNewPrivileges` 等 guardrail，且 service `inactive`、`MainPID=0`、未 enabled / 未 masked 時，才可保留 drain config / binary / unit，並輸出 `CONTROLLED_DRAIN_STAGING_ALLOWED=1` 與 textfile metric。此 staging 規則不得讀 token、不得讀 `.runner` 內容、不得註冊 runner、不得啟動 service；若 registration 缺失，readiness verifier 仍必須只留下 `controlled_cd_lane_registration_missing` / `controlled_cd_lane_service_not_active` 類 blocker。若 `CONTROLLED_DRAIN_STAGING_ALLOWED=0` 且 config / binary 又被搬走，優先修 source enforcer / unit guardrail，不要手工反覆補同一組 artifact。 diff --git a/docs/workplans/2026-07-02-commander-inserted-requirements-priority-ledger.md b/docs/workplans/2026-07-02-commander-inserted-requirements-priority-ledger.md index 79086c7c0..d4ad619d0 100644 --- a/docs/workplans/2026-07-02-commander-inserted-requirements-priority-ledger.md +++ b/docs/workplans/2026-07-02-commander-inserted-requirements-priority-ledger.md @@ -61,10 +61,10 @@ | 2 | CIR-P0-RBT-002 | P0 | 「沒有偵測到主機重啟」 | 修正 host reboot/shutdown/up detection：boot_id / uptime / node exporter / Windows exporter / VMware VM power state 都要進同一事件 | Scorecard 已接 collection packet + management probe；99 host reachable 但 uptime unknown，111 unreachable，stale hosts 仍存在 | 讓 99 verifier / Windows exporter 或等效 no-secret readback 進入 host boot event，並補 111 reachability 證據 | | 3 | CIR-P0-RBT-003 | P0 | 「192.168.0.99 VMWare 要自動啟動，裡面 111/188/120/121/112 也自動啟動」 | Windows 99 VMware host autostart + guest VM autostart contract；VM host 111/188/120/121/112 開機順序與 readback | Source verifier / parser / API readback / collection packet 已完成；management probe 讀回 `host_reachable=true`、RDP open、`2179` VMConnect / console channel open、SSH BatchMode `permission_denied`、WinRM timeout；snapshot active blockers=`windows99_remote_execution_channel_unavailable`、`windows99_vmware_autostart_readback_missing` | 收集 local console Verify output 或恢復 no-secret management channel，再確認 `VMRUN_PRESENT`、scheduled task、VMware services、VM power、VMX present 全綠 | | 4 | CIR-P0-RBT-004 | P0 | 「192.168.0.99 不可因 Windows Update 無預警重開」 | Windows Update reboot policy：active hours / no auto-restart / maintenance window / update notification audit | Source verifier 已補 `WINDOWS_UPDATE_POLICY` 與 `WINDOWS_UPDATE_NO_AUTO_REBOOT_READY`；collection packet 已列 forbidden actions；99 management channel 尚不能收 policy readback | 取得 Verify output；若 policy 不綠，再走 controlled apply，禁止要求或記錄 Windows 密碼 | -| 5 | CIR-P0-RBT-005 | P0 | 「網站重啟後 502 嚴重影響體驗，要維護頁，外部雲端或專業做法」 | Public maintenance fallback：Nginx / edge / external static maintenance page / status page / fail-open UX，避免 502 直出 | Source + runtime verifier 已實作；AWOOOI Nginx source conf 已從 legacy `/50x.html` 改成 L0 `proxy_intercept_errors on` + `__awoooi-maintenance.html` + `X-AWOOOI-Fallback=local-maintenance`；production post-rollout probe 目前 6 條 public routes 都是 `200`、raw 5xx=`0`；API runtime artifact overlay 已本機驗證可將 scorecard blockers `12 -> 11`、readiness `40 -> 47`，但 production API 尚未 CD 讀回 | 先推 Gitea CD 並讀回 `/api/v1/agents/reboot-auto-recovery-slo-scorecard` 清掉 `public_maintenance_fallback_runtime_readback_missing`；再做 edge staging vhost / `nginx -t` / header verifier；通過後才 controlled reload；若 edge unreachable，產生 L1 external static origin / CDN decision record + rollback | -| 6 | CIR-P0-RBT-006 | P0 | 「所有主機關機立刻 Telegram 告警，重啟後也要告警，其他告警一併完整思考」 | Down / shutdown suspected / reboot detected / reboot recovered / SLO missed / backup failed / freshness stale / CPU pressure / Gitea queue 告警矩陣 | 部分已有 Alertmanager rule 與 Telegram receipt 補強；仍缺完整 shutdown/up E2E receipt | 建立 Telegram alert matrix + receipt verifier，逐項讀回 Alertmanager active/resolved 與 outbound receipt，不送測試 secret | +| 5 | CIR-P0-RBT-005 | P0 | 「網站重啟後 502 嚴重影響體驗，要維護頁，外部雲端或專業做法」 | Public maintenance fallback：Nginx / edge / external static maintenance page / status page / fail-open UX，避免 502 直出 | Source + runtime verifier 已實作；Gitea CD `#4459` Success、deploy marker `8d7a6faaf`，production API scorecard 已讀回 `public_maintenance_fallback.ready=true`、raw 5xx=`0`、unreachable without L1=`0`，P0 blockers `11`、readiness `47`；仍不可宣稱整體 10 分鐘 SLO 完成 | 若未來 public route 出現 raw 5xx，先走 L0 Nginx intercept / header verifier；若 edge unreachable，才產生 L1 external static origin / CDN decision record + rollback | +| 6 | CIR-P0-RBT-006 | P0 | 「所有主機關機立刻 Telegram 告警，重啟後也要告警，其他告警一併完整思考」 | Down / shutdown suspected / reboot detected / reboot recovered / SLO missed / backup failed / freshness stale / CPU pressure / Gitea queue 告警矩陣 | HostDown / HostRebootEventDetected / RebootAutoRecoverySLOMissed 已存在；本輪補 `awoooi_reboot_auto_recovery_slo_active_blocker{blocker=...}`、`RebootAutoRecoveryActiveBlocker` 與 metric-missing guard，避免只看到總 SLO 告警；仍需完整 shutdown/up E2E receipt | 部署 alert rules 後讀回 Prometheus / Alertmanager active/resolved 與 outbound receipt；不送測試 secret，不重啟主機 | | 7 | CIR-P0-RBT-007 | P0 | 「所有備份包含主機、DB、網站、服務、套件、工具、日誌都沒有監控告警」 | Backup observability coverage：backup job inventory、last success、freshness、offsite、restore drill、Telegram receipt | 部分已有 backup health exporter / alert rules；全域 coverage 與 restore drill 未全綠 | 建立 backup coverage matrix：host / DB / website / service config / package list / tool scripts / logs，每列有 metric、alert、last_success、restore_verifier | -| 8 | CIR-P0-RBT-008 | P0 | 「每次重啟排查都不一樣，也不知道多久恢復，不符合 SLA」 | 固定化 reboot runbook：fixed triage order、ETA、active blocker、remaining seconds、owner lane、next command | Production scorecard readback 已固定 `status=blocked_reboot_auto_recovery_slo_not_ready`、readiness `40%`、active blockers `12`；本機套疊 live public maintenance artifact 後可降為 readiness `47%`、active blockers `11`；`next_safe_action=restore_windows99_no_secret_management_channel_or_collect_local_console_verify_readback_then_rerun_reboot_scorecard_no_reboot` | 先完成 API overlay CD / production readback，再優先收斂 99 no-secret Verify / 111 reachability / backup_core / service_green / Wazuh degraded；不得用不同排查路徑繞過 scorecard | +| 8 | CIR-P0-RBT-008 | P0 | 「每次重啟排查都不一樣，也不知道多久恢復，不符合 SLA」 | 固定化 reboot runbook：fixed triage order、ETA、active blocker、remaining seconds、owner lane、next command | Production scorecard readback 已固定 `status=blocked_reboot_auto_recovery_slo_not_ready`、readiness `47%`、active blockers `11`；本輪再補 per-blocker Prometheus metric / Telegram alert 投影，避免只剩總 blocker_count；`next_safe_action=restore_windows99_no_secret_management_channel_or_collect_local_console_verify_readback_then_rerun_reboot_scorecard_no_reboot` | 優先收斂 99 no-secret Verify / 111 reachability / backup_core / service_green / Wazuh degraded；不得用不同排查路徑繞過 scorecard | | 9 | CIR-P0-RBT-009 | P0 | 「所有產品、網站都要是最新版本；版本和數據是否最新要驗證」 | Product freshness/version matrix：source commit、deploy marker、runtime image、public health、data freshness、latest source availability | AWOOOI Gitea main 已到 deploy marker `c68b74686`，production source readback `5d5bc86fed` verified；StockPlatform public freshness / ingestion 讀回 `ok`，latest trading date `2026-07-02`，core price/chips/margin/AI recommendations 都是 `2026-07-02` | 建立全產品 readback 表：product、canonical repo、main SHA、deploy marker、public URL、data freshness、blocked reason | | 10 | CIR-P0-GIT-001 | P0 | 「Gitea 儲存庫都不見了？Gitea 沒完整備份嗎？」 | Gitea repository identity + backup proof + restore drill：不能只看 UI visible，要比對 SSH heads、repo path、bundle backup、restore sample | 已有 9 expected repos OK / backup health missing=0 的 handoff；仍需 restore drill 證明 | 補 Gitea repo bundle backup readback + sample restore dry-run verifier；禁止刪 repo / 改 visibility | | 11 | CIR-P0-CPU-001 | P0 | 「110 / 188 CPU 負載持續過高，為什麼沒監控告警、沒主動修復」 | Sustained CPU pressure automation：Alertmanager → controller → evidence → service playbook → verifier → KM writeback | 110 已有 `Host110SustainedModeratePressure`、Gitea playbook、Stock/Postgres evidence；188 仍需同級 controller/alerts readback | 下一步接 `postgres_hot_query_or_backup_export_playbook`；並補 188 equivalent readback，不以單次下降結案 | @@ -112,6 +112,8 @@ | 110 CPU evidence / controller 分流一致性 | live evidence 與 controller 皆將 Stock/Postgres pressure 優先導向 `postgres_hot_query_or_backup_export_playbook` | | 插入需求 API/UI 可見化 | `awoooi-priority-work-order-readback` 已把插入需求主線工作項產品化；Work Items 頁有 spotlight | | Telegram alert AI Loop post-apply verifier | `telegram-alert-learning-context-post-apply-verifier` API / service / tests 已完成，AI automation `post_verifier` 節點可讀回 verified context receipts | +| Public maintenance fallback runtime readback | Gitea CD `#4459` / deploy marker `8d7a6faaf` 已讓 production scorecard 讀回 `public_maintenance_fallback.ready=true`、raw 5xx=`0`、P0 blockers `11`、readiness `47` | +| Reboot SLO per-blocker 告警投影 | Source 已補 `awoooi_reboot_auto_recovery_slo_active_blocker{blocker=...}`、`RebootAutoRecoveryActiveBlocker`、`RebootAutoRecoveryActiveBlockerMetricMissing` 與契約測試 | ### In Progress @@ -132,7 +134,7 @@ | AI 專業 UI / 非文字牆 cockpit | 已列為 P2 UX 驗收 | 將長文字區塊收斂成 first-viewport cockpit、cards、flow rows 與 expandable details | | 10 分鐘 reboot auto-recovery SLA | scorecard source / snapshot / API 已補固定 phase / ETA / blocker / next action 欄位；仍缺 fresh all-host reboot/drill proof | 補 production API readback，缺事件則明確標等待下一次 reboot 或 approved drill | | 99 Windows / VMware autostart | Source verifier / parser / API readback / collection packet 已完成；live 99 Verify output 尚未收集，scorecard 以 `windows99_vmware_autostart_readback_missing` fail-closed；collection packet 顯示 99 reachable、可收 no-secret Verify，但 uptime unknown | 收集 99 no-secret Verify output，確認 VM 111/188/120/121/112 running、scheduled task / services / Windows Update policy 全綠 | -| 502 maintenance fallback | Source + scorecard verifier 已補；API runtime artifact overlay 已本機驗證可清掉 `public_maintenance_fallback_runtime_readback_missing`；真實 L0/L1 runtime 套用仍待 edge readback，不可宣稱完成 | 推 Gitea CD 後讀回 production API；raw 5xx 則推 L0 Nginx intercept，edge unreachable 則推 L1 external static fallback decision + rollback | +| Reboot SLO blocker 收斂 | production scorecard 已固定 `11` blockers / readiness `47`，且 source 已補 per-blocker metric/alert；還沒部署本輪告警規則 | 推 Gitea CD 後部署/讀回 Prometheus rules，再依具名 blocker 收斂 99/111/backup/service/Wazuh | | 全備份監控告警 coverage | 部分 exporter/rule 已存在，但 host/DB/site/service/package/tool/log coverage 未全列 | 建立 backup coverage matrix 與 restore drill verifier | | Stock/Postgres hot pressure | 110 live 已導向 Stock/Postgres playbook；尚未完成 hot query / backup export playbook closure | 下一步執行 read-only Stock/Postgres evidence 與 source freshness / query attribution | @@ -149,13 +151,13 @@ ## 下一輪固定執行順序 1. 先確認 Gitea `main` / CD / production readback 是否仍與最新 truth 一致。 -2. 接續目前 active P0-006：把 public maintenance runtime artifact overlay 推進到 Gitea CD，讀回 production API scorecard，確認 blockers `12 -> 11`、readiness `40 -> 47`。 -3. 補全 reboot auto-recovery P0：99/110/111/112/120/121/188 reboot detection、10 分鐘 SLO scorecard、Telegram down/up/recovered/SLO missed receipts。 +2. 接續目前 active P0-006：把 per-blocker reboot SLO metric / alert 修法推進到 Gitea CD，讀回 Prometheus / Alertmanager / production scorecard。 +3. 補全 reboot auto-recovery P0：99/110/111/112/120/121/188 reboot detection、10 分鐘 SLO scorecard、Telegram down/up/recovered/SLO missed / active-blocker receipts。 4. 補 99 Windows / VMware autostart P0：99 host 自動啟動 VMware，VM 111/188/120/121/112 自動啟動；同時補 Windows Update no-auto-restart verifier。 5. 補 111 reachability / host boot readback，讓 `host_unreachable_after_reboot` 與 `reboot_event_required_host_unreachable` 有單一固定處理路徑。 6. 補 backup observability P0：host / DB / website / service config / package list / tools / logs backup matrix、last success、offsite、restore drill、Telegram receipt。 7. 補 service_green / Wazuh degraded readback，清楚分開 reboot service blocker 與 P2 observability debt。 -8. 補 public maintenance fallback P0：production API overlay 清掉缺讀回後，再做 edge staging vhost / `nginx -t` / header verifier；raw 502 走 L0，edge unreachable 走 L1 external static fallback decision + rollback。 +8. 補 public maintenance fallback P0：production 已清掉缺讀回；後續只在 raw 502 再現時走 L0 Nginx intercept / header verifier，edge unreachable 才走 L1 external static fallback decision + rollback。 9. 補 product freshness/version P0：所有產品網站 source SHA / deploy marker / runtime image / public URL / data freshness readback。 10. 接續 110 / 188 CPU pressure P0：跑 `postgres_hot_query_or_backup_export_playbook` 的 read-only evidence / source freshness / query attribution，並補 188 equivalent readback。 11. 繼續補 110 controlled drain lane 的 staging / verifier，不恢復 generic runner。 diff --git a/ops/monitoring/alerts-unified.yml b/ops/monitoring/alerts-unified.yml index d923052a9..3c63bd337 100644 --- a/ops/monitoring/alerts-unified.yml +++ b/ops/monitoring/alerts-unified.yml @@ -210,6 +210,41 @@ groups: description: "偵測到重啟事件，但 10 分鐘期限已過且 awoooi_reboot_auto_recovery_slo_ready 仍為 0。" runbook: "直接讀最新 scorecard.json 的 active_blockers 與 fixed_triage_order；先處理 99/VMware/VM power、public maintenance fallback、registry/K3s、Stock freshness、backup health 與 Telegram delivery。" + - alert: RebootAutoRecoveryActiveBlocker + expr: awoooi_reboot_auto_recovery_slo_active_blocker{scope="99_110_111_112_120_121_188"} == 1 + for: 2m + labels: + severity: critical + layer: full-stack + component: reboot-auto-recovery-slo + team: ops + alert_category: reboot_recovery + notification_type: TYPE-3 + auto_repair: "false" + annotations: + summary: "重啟恢復 SLO blocker 未清除：{{ $labels.blocker }}" + description: "reboot-auto-recovery scorecard 仍有 active blocker={{ $labels.blocker }}；不可只看總 blocker_count，必須對應處理 99/VMware、主機可達、service/backup/Wazuh 或 public fallback 的具體證據。" + runbook: "先讀 110 最新 /home/wooo/reboot-recovery/reboot-auto-recovery-slo-*/scorecard.json、summary.txt、host-probe.txt 與同 run 的 public-maintenance/windows99 readback；Windows99 只走 no-secret collector 或 console verify，backup 只讀 backup-status/exporter，Wazuh 只讀 registry/API；禁止 reboot、Docker/systemd/Nginx/DB/firewall restart、restore/prune/delete。" + + - alert: RebootAutoRecoveryActiveBlockerMetricMissing + expr: | + (awoooi_reboot_auto_recovery_slo_blocker_count{scope="99_110_111_112_120_121_188"} > 0) + unless on(scope, target_minutes) + (count by(scope, target_minutes) (awoooi_reboot_auto_recovery_slo_active_blocker{scope="99_110_111_112_120_121_188"}) > 0) + for: 5m + labels: + severity: warning + layer: systemd-110 + component: reboot-auto-recovery-slo + team: ops + alert_category: reboot_recovery + notification_type: TYPE-1 + auto_repair: "false" + annotations: + summary: "重啟恢復 SLO 只有 blocker 總數，缺少具名 blocker metric" + description: "awoooi_reboot_auto_recovery_slo_blocker_count > 0，但沒有 awoooi_reboot_auto_recovery_slo_active_blocker；Telegram 只能看到總數，無法自動指出真問題。" + runbook: "部署新版 reboot-auto-recovery-slo-exporter.sh，確認 node-exporter textfile 中有 awoooi_reboot_auto_recovery_slo_active_blocker{blocker=...}，再重跑 Prometheus rule drift guard。" + - alert: RebootAutoRecoverySLOExporterStale expr: time() - awoooi_reboot_auto_recovery_slo_last_run_timestamp{scope="99_110_111_112_120_121_188"} > 600 for: 10m diff --git a/scripts/ops/tests/test_host_pressure_alert_contract.py b/scripts/ops/tests/test_host_pressure_alert_contract.py index f39ec7a9c..d18282895 100644 --- a/scripts/ops/tests/test_host_pressure_alert_contract.py +++ b/scripts/ops/tests/test_host_pressure_alert_contract.py @@ -60,3 +60,21 @@ def test_backup_aggregate_alert_excludes_old_wrapper_noise() -> None: assert 'awoooi_backup_last_run_failed_count{host="110",exported_job!="backup_all"}' in expr assert 'exported_job="backup_all"} > 0' not in expr + + +def test_reboot_slo_alerts_project_named_blockers_to_telegram() -> None: + alerts = load_alerts() + blocker = alerts["RebootAutoRecoveryActiveBlocker"] + metric_missing = alerts["RebootAutoRecoveryActiveBlockerMetricMissing"] + + assert "awoooi_reboot_auto_recovery_slo_active_blocker" in str(blocker["expr"]) + assert blocker["labels"]["notification_type"] == "TYPE-3" + assert "$labels.blocker" in blocker["annotations"]["summary"] + assert "$labels.blocker" in blocker["annotations"]["description"] + assert "Windows99" in blocker["annotations"]["runbook"] + assert "backup-status" in blocker["annotations"]["runbook"] + assert "禁止 reboot" in blocker["annotations"]["runbook"] + + assert "awoooi_reboot_auto_recovery_slo_blocker_count" in str(metric_missing["expr"]) + assert "awoooi_reboot_auto_recovery_slo_active_blocker" in str(metric_missing["expr"]) + assert metric_missing["labels"]["notification_type"] == "TYPE-1" diff --git a/scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh b/scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh index cd1e2066c..2ad93d2e3 100755 --- a/scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh +++ b/scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh @@ -149,6 +149,38 @@ payload=json.load(open(sys.argv[1], encoding="utf-8")) print(payload.get("public_maintenance_fallback", {}).get("route_unreachable_without_external_fallback_count", 0) or 0) PY )" +active_blocker_metrics="$(python3 - "$scorecard_file" "$TARGET_MINUTES" <<'PY' +import json, sys + +payload = json.load(open(sys.argv[1], encoding="utf-8")) +target_minutes = str(sys.argv[2]) +scope = "99_110_111_112_120_121_188" + + +def label(value: object) -> str: + text = str(value or "") + return ( + text.replace("\\", "\\\\") + .replace("\n", "\\n") + .replace('"', '\\"') + ) + + +for blocker in payload.get("active_blockers") or []: + if blocker: + print( + "awoooi_reboot_auto_recovery_slo_active_blocker" + f'{{scope="{scope}",target_minutes="{label(target_minutes)}",blocker="{label(blocker)}"}} 1' + ) + +primary_blocker = payload.get("primary_blocker") or "" +if primary_blocker: + print( + "awoooi_reboot_auto_recovery_slo_primary_blocker" + f'{{scope="{scope}",target_minutes="{label(target_minutes)}",blocker="{label(primary_blocker)}"}} 1' + ) +PY +)" tmp_metric="$(mktemp "$TEXTFILE_DIR/.reboot_auto_recovery_slo.XXXXXX")" cat >"$tmp_metric" <>"$tmp_metric" +fi if [ -s "$reboot_event_prom" ]; then cat "$reboot_event_prom" >>"$tmp_metric" fi diff --git a/scripts/reboot-recovery/reboot-recovery-readiness-audit.sh b/scripts/reboot-recovery/reboot-recovery-readiness-audit.sh index 7e5282203..ecdcc095b 100755 --- a/scripts/reboot-recovery/reboot-recovery-readiness-audit.sh +++ b/scripts/reboot-recovery/reboot-recovery-readiness-audit.sh @@ -343,6 +343,10 @@ require_pattern "188-internal-tools-https.conf.j2" infra/ansible/playbooks/nginx echo echo "== Monitoring and CI ==" +require_pattern "awoooi_reboot_auto_recovery_slo_active_blocker" scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh "Reboot SLO active blocker textfile metric" +require_pattern "awoooi_reboot_auto_recovery_slo_primary_blocker" scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh "Reboot SLO primary blocker textfile metric" +require_pattern "RebootAutoRecoveryActiveBlocker" ops/monitoring/alerts-unified.yml "Reboot SLO active blocker alert" +require_pattern "RebootAutoRecoveryActiveBlockerMetricMissing" ops/monitoring/alerts-unified.yml "Reboot SLO active blocker metric missing alert" require_pattern "cold_start_recovery_alerts" ops/monitoring/alerts-unified.yml "Cold-start alert group" require_pattern "PrometheusRuleDriftGuardFailed" ops/monitoring/alerts-unified.yml "Prometheus rule drift guard failure alert" require_pattern "PrometheusRuleDriftAutoRepaired" ops/monitoring/alerts-unified.yml "Prometheus rule drift repaired alert" diff --git a/scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_installer.py b/scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_installer.py index fcbea5514..593f981a9 100644 --- a/scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_installer.py +++ b/scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_installer.py @@ -97,3 +97,14 @@ def test_exporter_carries_stockplatform_readbacks_into_scorecard() -> None: assert "scorecard_args+=(--stock-ingestion-file" in text assert "manual_db_update" not in text assert "systemctl restart" not in text + + +def test_exporter_projects_each_scorecard_blocker_to_textfile_metric() -> None: + text = EXPORTER.read_text(encoding="utf-8") + + assert "active_blocker_metrics" in text + assert 'for blocker in payload.get("active_blockers") or []' in text + assert "awoooi_reboot_auto_recovery_slo_active_blocker" in text + assert "awoooi_reboot_auto_recovery_slo_primary_blocker" in text + assert 'blocker="{label(blocker)}"' in text + assert 'target_minutes="{label(target_minutes)}"' in text diff --git a/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py b/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py index bd61de45a..738c825d6 100644 --- a/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py +++ b/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py @@ -47,6 +47,8 @@ def test_reboot_p0_contract_has_maintenance_and_telegram_alerts() -> None: for alert_name in [ "HostRebootEventDetected", "RebootAutoRecoverySLOMissed", + "RebootAutoRecoveryActiveBlocker", + "RebootAutoRecoveryActiveBlockerMetricMissing", "PublicRouteServing5xxWithoutMaintenanceFallback", "BackupCoverageDomainStale", ]: @@ -61,6 +63,9 @@ def test_reboot_p0_contract_has_maintenance_and_telegram_alerts() -> None: assert "public-maintenance-fallback.json" in exporter assert "--public-maintenance-file" in exporter assert "awoooi_public_route_raw_5xx_without_maintenance_fallback" in exporter + assert "awoooi_reboot_auto_recovery_slo_active_blocker" in exporter + assert 'blocker="{label(blocker)}"' in exporter + assert "$labels.blocker" in alerts def test_reboot_p0_contract_applies_l0_fallback_to_awoooi_nginx_sources() -> None: