From a31a278e585d9a186d2110495ea856df98504bfa Mon Sep 17 00:00:00 2001 From: ogt Date: Wed, 15 Jul 2026 02:05:14 +0800 Subject: [PATCH] fix(recovery): align host112 cold-start metric scope --- .../services/agent99_same_run_reconcile.py | 22 +++---- .../tests/test_agent99_same_run_reconcile.py | 11 ++++ docs/runbooks/FULL-STACK-COLD-START-SOP.md | 14 +++-- docs/runbooks/REBOOT-RECOVERY-SOP.md | 12 +++- .../roles/cold-start-monitor/tasks/main.yml | 2 +- ops/monitoring/alerts-unified.yml | 24 ++++---- .../full-stack-cold-start-baseline.yml | 2 +- .../ops/recovery-scorecard-contract-check.py | 2 +- .../full-stack-recovery-scorecard.sh | 6 +- .../test_cold_start_monitor_bounded_probes.py | 17 ++++++ .../verify-cold-start-monitor-deploy.sh | 58 +++++++++++++++++++ 11 files changed, 134 insertions(+), 36 deletions(-) diff --git a/apps/api/src/services/agent99_same_run_reconcile.py b/apps/api/src/services/agent99_same_run_reconcile.py index ed4e10555..bc16244fc 100644 --- a/apps/api/src/services/agent99_same_run_reconcile.py +++ b/apps/api/src/services/agent99_same_run_reconcile.py @@ -38,41 +38,41 @@ AGENT99_SAME_RUN_STATUS_VERIFIER = ( "agent99-status-same-run-no-write-post-condition-v1" ) AGENT99_COLD_START_METRIC_HOST = "110" -AGENT99_COLD_START_METRIC_SCOPE = "110_120_121_188" +AGENT99_COLD_START_METRIC_SCOPE = "110_112_120_121_188" AGENT99_COLD_START_MAX_AGE_SECONDS = 900 _COLD_START_QUERIES = { "monitor_up": ( - 'awoooi_cold_start_monitor_up{host="110",scope="110_120_121_188",' + f'awoooi_cold_start_monitor_up{{host="110",scope="{AGENT99_COLD_START_METRIC_SCOPE}",' 'mode="read_only"}' ), "pass_gates": ( - 'awoooi_cold_start_pass_gates{host="110",scope="110_120_121_188"}' + f'awoooi_cold_start_pass_gates{{host="110",scope="{AGENT99_COLD_START_METRIC_SCOPE}"}}' ), "warn_gates": ( - 'awoooi_cold_start_warn_gates{host="110",scope="110_120_121_188"}' + f'awoooi_cold_start_warn_gates{{host="110",scope="{AGENT99_COLD_START_METRIC_SCOPE}"}}' ), "blocked_gates": ( - 'awoooi_cold_start_blocked_gates{host="110",scope="110_120_121_188"}' + f'awoooi_cold_start_blocked_gates{{host="110",scope="{AGENT99_COLD_START_METRIC_SCOPE}"}}' ), "last_run_timestamp": ( - 'awoooi_cold_start_last_run_timestamp{host="110",' - 'scope="110_120_121_188"}' + f'awoooi_cold_start_last_run_timestamp{{host="110",' + f'scope="{AGENT99_COLD_START_METRIC_SCOPE}"}}' ), "green_result": ( - 'awoooi_cold_start_last_result{host="110",scope="110_120_121_188",' + f'awoooi_cold_start_last_result{{host="110",scope="{AGENT99_COLD_START_METRIC_SCOPE}",' 'result="green"}' ), "degraded_result": ( - 'awoooi_cold_start_last_result{host="110",scope="110_120_121_188",' + f'awoooi_cold_start_last_result{{host="110",scope="{AGENT99_COLD_START_METRIC_SCOPE}",' 'result="degraded"}' ), "blocked_result": ( - 'awoooi_cold_start_last_result{host="110",scope="110_120_121_188",' + f'awoooi_cold_start_last_result{{host="110",scope="{AGENT99_COLD_START_METRIC_SCOPE}",' 'result="blocked"}' ), "check_failed_result": ( - 'awoooi_cold_start_last_result{host="110",scope="110_120_121_188",' + f'awoooi_cold_start_last_result{{host="110",scope="{AGENT99_COLD_START_METRIC_SCOPE}",' 'result="check_failed"}' ), "firing_blocking_alerts": ( diff --git a/apps/api/tests/test_agent99_same_run_reconcile.py b/apps/api/tests/test_agent99_same_run_reconcile.py index 78f92f100..4fae7764b 100644 --- a/apps/api/tests/test_agent99_same_run_reconcile.py +++ b/apps/api/tests/test_agent99_same_run_reconcile.py @@ -117,6 +117,17 @@ def _evidence_refs() -> dict[str, str]: } +def test_cold_start_source_queries_include_host112_scope() -> None: + expected_scope = 'scope="110_112_120_121_188"' + + assert reconcile.AGENT99_COLD_START_METRIC_SCOPE == "110_112_120_121_188" + for name, query in reconcile._COLD_START_QUERIES.items(): + if name == "firing_blocking_alerts": + continue + assert expected_scope in query + assert 'scope="110_120_121_188"' not in query + + def test_cold_start_source_resolution_allows_warning_only_but_requires_fresh_zero( monkeypatch, ) -> None: diff --git a/docs/runbooks/FULL-STACK-COLD-START-SOP.md b/docs/runbooks/FULL-STACK-COLD-START-SOP.md index 647a0d9b7..162aca170 100644 --- a/docs/runbooks/FULL-STACK-COLD-START-SOP.md +++ b/docs/runbooks/FULL-STACK-COLD-START-SOP.md @@ -1,7 +1,7 @@ # AWOOOI 全棧冷啟動與主機重啟 SOP -> Version: v1.136 -> Last updated: 2026-07-14 Asia/Taipei +> Version: v1.137 +> Last updated: 2026-07-15 Asia/Taipei > Scope: 99 / 110 / 111 / 112 / 120 / 121 / 188 全棧重啟恢復。112 仍是 Kali / VM guest 訊號，但 2026-06-30 全主機重啟後已納入 10 分鐘 SLO 的必要 boot / power signal；此納入不代表授權任何破壞性 runtime apply。 --- @@ -10,6 +10,12 @@ 本節是每次接手、開機、關機、重啟後的第一個判定錨點。若日期不是今天，必須先重跑 live check，再更新本節與 `docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md`。 +### v1.137 2026-07-15 Cold-start metric scope 全鏈路防漂移 + +Cold-start exporter 的 canonical scope 固定為 `110_112_120_121_188`。producer、`ops/monitoring/alerts-unified.yml` 的 recording/alert rules、Agent99 same-run reconciler、full-stack recovery scorecard、baseline 與 Ansible post-verifier 必須使用同一個 label value；任一 consumer 仍查詢舊 `110_120_121_188` 時，會把正常的新 series 誤判成缺失並產生假的 `ColdStartMonitorMissing`，因此只能標示 monitoring contract drift，不能觸發主機或服務重啟。 + +每次 exporter scope 或 required host inventory 變更後，`verify-cold-start-monitor-deploy.sh` 除了 script hash、textfile 與 host112 blocker series，也必須讀 Prometheus `/api/v1/rules` 驗證所有 cold-start recording/alert expressions已切到 canonical scope、舊 scope 為零，並確認 reload 後 `ColdStartMonitorMissing` 不再 firing。禁止人工 silence 或刪除告警掩蓋 drift；先修 canonical source、走受控 rules deploy/reload，再以 Prometheus rule readback、recovery scorecard 與 Agent99 same-run source verifier 共同收斂。 + ### v1.136 2026-07-14 Host112 正式納入 cold-start P0 release gate `full-stack-cold-start-check.sh` 舊版仍顯示 `112 Kali is intentionally skipped`，與 Windows99 五台 required VM、Agent99 `host112_guest_readiness` phase 及全主機 10 分鐘 SLO 不一致。現行 baseline 已把 112 從 excluded host 移入 included host；P0 network 固定檢查 ping / TCP 22，並透過 `kali@192.168.0.112` 執行 canonical `/usr/local/sbin/awoooi-host112-guest-readiness --check`。release gate 必須同時驗證 readback schema、systemd running、guest/console/services/recovery timer、Wazuh 1514/1515/55000 independent verifier 與 `runtime_write_performed=0`。任何一項失敗均維持 BLOCKED；不得因其他四台 Linux host 或 public routes 為 200 而宣稱全主機恢復。110 textfile exporter scope 固定為 `110_112_120_121_188`，並分開輸出 112 host unreachable 與 guest readiness failed blocker series；`verify-cold-start-monitor-deploy.sh` 必須驗證 source/live hash、scope 與 blocker series parity，避免 checker 已納管但 Prometheus 仍漏掉 112。 @@ -67,7 +73,7 @@ v1.80 / v1.81 credential escrow intake scorecard rule：同一輪 owner response v1.93 reboot SLA readback fixed fields rule：`scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py`、`docs/operations/awoooi-reboot-auto-recovery-slo-scorecard.snapshot.json` 與 `/api/v1/agents/reboot-auto-recovery-slo-scorecard` 必須固定輸出 `current_phase`、`eta_or_wait_reason`、`primary_blocker`、`active_blockers`、`next_safe_action` 與 `reboot_sop_progress.fixed_triage_order`。回報時先讀這些欄位，不得重新臨場發明排查順序；若 `current_phase=host_boot_detection_blocked` 且 `eta_or_wait_reason=target_window_elapsed_eta_unavailable_until_fresh_all_host_reboot_event_and_probe`，只能宣稱「10 分鐘 SLO 尚未證明」，下一步固定補 fresh all-host reboot event / probe 與 99 VMware/Windows readback，不得把部分 Linux host green 當全主機 green。 -v1.94 Windows 99 / VMware verifier rule：99 主機與 guest VM 自動啟動不得再只靠「ping 到 99」或「VMX source 存在」宣稱。`scripts/reboot-recovery/windows99-vmware-autostart.ps1 -Mode Verify` 必須輸出 no-secret 固定欄位：`VMRUN_PRESENT`、`VMX alias=... present=...`、`VMWARE_SERVICE ... ok=...`、`VMWARE_AUTOSTART_TASK ... ok=...`、`WINDOWS_UPDATE_POLICY ... ok=...`、`VM_POWER alias=... running=...`、`VMWARE_AUTOSTART_CONFIG_READY`、`VMWARE_AUTOSTART_POWER_READY`、`WINDOWS_UPDATE_NO_AUTO_REBOOT_READY`、`VMWARE_AUTOSTART_VERIFY_READY`。`reboot-auto-recovery-slo-scorecard.py --windows99-vmware-file ` 必須解析這些欄位；缺 readback 時 active blocker 固定為 `windows99_vmware_autostart_readback_missing`，next action 固定為 `collect_windows99_vmware_autostart_verify_readback_then_rerun_all_host_reboot_scorecard_no_secret_no_reboot`。預設 required guest VM alias 以本輪 reboot 事故 source-of-truth 為準：`111 / 188 / 120 / 121 / 112`；`110` 保留為 required host / control-plane reachability，不得被當成 99 VMware 預設 VM。此 verifier 不讀 Windows 密碼、不讀 secret、不啟動 VM、不重啟 host；`Apply` 仍需獨立 controlled apply 與 post-verifier。 +v1.94 Windows 99 / VMware verifier rule：99 主機與 guest VM 自動啟動不得再只靠「ping 到 99」或「VMX source 存在」宣稱。`scripts/reboot-recovery/windows99-vmware-autostart.ps1 -Mode Verify` 必須輸出 no-secret 固定欄位：`VMRUN_PRESENT`、`VMX alias=... present=...`、`VMWARE_SERVICE ... ok=...`、`VMWARE_AUTOSTART_TASK ... ok=...`、`WINDOWS_UPDATE_POLICY ... ok=...`、`VM_POWER alias=... running=...`、`VMWARE_AUTOSTART_CONFIG_READY`、`VMWARE_AUTOSTART_POWER_READY`、`WINDOWS_UPDATE_NO_AUTO_REBOOT_READY`、`VMWARE_AUTOSTART_VERIFY_READY`。`reboot-auto-recovery-slo-scorecard.py --windows99-vmware-file ` 必須解析這些欄位；缺 readback 時 active blocker 固定為 `windows99_vmware_autostart_readback_missing`，next action 固定為 `collect_windows99_vmware_autostart_verify_readback_then_rerun_all_host_reboot_scorecard_no_secret_no_reboot`。現行 required guest VM alias 依 live inventory 固定為 `110 / 188 / 120 / 121 / 112`；`111` 是獨立 Darwin 主機，不得被當成 Windows99 VM。舊 `111 / 188 / 120 / 121 / 112` alias contract 已由 `global_product_governance_v2`、v1.133 與本規則取代。此 verifier 不讀 Windows 密碼、不讀 secret、不啟動 VM、不重啟 host；`Apply` 仍需獨立 controlled apply 與 post-verifier。 v1.95 Windows 99 management-channel readback rule：若 99 可 ping / RDP / TCP，但 `windows99-vmware-autostart.ps1 -Mode Verify` 尚未收回，不得只寫成「等 verifier」。必須先跑 `scripts/reboot-recovery/windows99-management-channel-probe.py --output `，並把結果用 `reboot-auto-recovery-slo-scorecard.py --windows99-management-file ` 接入同一份 SLO scorecard。此 probe 只做 no-secret readback：TCP `22 / 135 / 445 / 2179 / 3389 / 5985 / 5986`、SSH BatchMode publickey、WinRM port、RDP console reachability 與 VMConnect / local console reachability；不得讀 Windows 密碼、不得啟動 VM、不得重啟、不得改 Windows Update。2026-07-02 18:28 live readback：`host_reachable=true`、`rdp_console_reachable=true`、`hyperv_vmconnect_open=true`、`local_console_channel_reachable=true`、`console_collection_channels=["rdp_console","hyperv_vmconnect"]`、`ssh_batch.status=permission_denied`、`winrm_http_open=false`、`winrm_https_open=false`、`remote_execution_channel_ready=false`；因此 active blockers 必須包含 `windows99_remote_execution_channel_unavailable` 與 `windows99_vmware_autostart_readback_missing`，next action 固定為 `restore_windows99_no_secret_management_channel_or_collect_local_console_verify_readback_then_rerun_reboot_scorecard_no_reboot`。 @@ -1426,7 +1432,7 @@ docker exec momo-pro-system python -c 'from services.cache_service import clear_ ## 14. 主機開機、關機、重啟 SOP -本節是每次 110 / 120 / 121 / 188 相關電源操作的標準程序。112 是 Kali，只保留 read-only evidence，不納入本輪恢復或例行重啟釋出。 +本節是每次 110 / 112 / 120 / 121 / 188 相關電源操作的標準程序。112 是 Windows99 上的 required Kali guest，必須通過 VM power、boot identity、SSH forced-command、systemd、console、Wazuh 與 recovery timer readback 才能釋出；舊版「112 不納入恢復」已由 `global_product_governance_v2` 與 v1.136 取代。 ### 14.1 共同紅線 diff --git a/docs/runbooks/REBOOT-RECOVERY-SOP.md b/docs/runbooks/REBOOT-RECOVERY-SOP.md index 6d6065ac6..e80956cc1 100644 --- a/docs/runbooks/REBOOT-RECOVERY-SOP.md +++ b/docs/runbooks/REBOOT-RECOVERY-SOP.md @@ -1,9 +1,9 @@ # AWOOOI 重開機恢復 SOP -> **版本**: v5.32 -> **最後更新**: 2026-07-14 (台北時間) +> **版本**: v5.33 +> **最後更新**: 2026-07-15 (台北時間) > **更新者**: Codex -> **觸發事件**: 2026-07-14 Agent99 host111/host112 readback false-alert closure +> **觸發事件**: 2026-07-15 cold-start metric scope / Agent99 same-run false-alert closure --- @@ -21,6 +21,12 @@ ## 架構概覽與依賴圖 +### 2026-07-15 Cold-start scope 全鏈路防漂移 + +Cold-start exporter、Prometheus recording/alert rules、Agent99 same-run reconciler、recovery scorecard、baseline 與 Ansible verifier 的 canonical metric scope 一律是 `110_112_120_121_188`。若 producer 已輸出此 scope，但任何 consumer 仍查詢舊 `110_120_121_188`，Prometheus 會產生假的 `ColdStartMonitorMissing`，Agent99 callback 也會因 source verifier 缺資料而 fail closed。這是監控合約漂移，不是主機離線；禁止以 silence、重啟主機或重啟產品服務處理。 + +受控修復順序固定為：source contract diff -> focused tests / rule parse -> Gitea CD/deploy marker -> canonical rules deploy/reload -> Prometheus `/api/v1/rules` expression readback -> firing alert resolution -> recovery scorecard -> Agent99 same-run callback verifier。`verify-cold-start-monitor-deploy.sh` 必須驗證 active rules 全部使用 canonical scope 且舊 scope 為零；缺任一 receipt 只能標示 partial/degraded，不能宣稱 Agent99 自動修復閉環完成。 + ### 2026-07-14 Agent99 host readback false-alert closure 111 是獨立 Darwin 主機，不是 Windows99 VM。Agent99 必須以 `ooo@192.168.0.111` 經 110 的 restricted forced-command route 讀取狀態；使用預設 `wooo` 會得到 SSH failure，不能包裝成 CPU 壓力。forced command 固定輸出 boot/Ollama 加 `agent99_perf_readback_v1` 四項唯讀指標，仍禁止 shell、PTY、forwarding、secret read 與任意命令。Windows99 deploy/bootstrap 必須自動遷移並 post-verify `sshUsers["192.168.0.111"]="ooo"`；Perf 只有在 cores/load/memory/disk 全部可解析時才可通過。 diff --git a/infra/ansible/roles/cold-start-monitor/tasks/main.yml b/infra/ansible/roles/cold-start-monitor/tasks/main.yml index 798250287..0cf34e28c 100644 --- a/infra/ansible/roles/cold-start-monitor/tasks/main.yml +++ b/infra/ansible/roles/cold-start-monitor/tasks/main.yml @@ -66,7 +66,7 @@ - name: "cold-start monitor | 驗證 green metric 存在" ansible.builtin.command: cmd: >- - grep -q 'awoooi_cold_start_last_result{host="110",scope="110_120_121_188",result="green"} 1' + grep -q 'awoooi_cold_start_last_result{host="110",scope="110_112_120_121_188",result="green"} 1' {{ cold_start_monitor_textfile_dir }}/cold_start_recovery.prom become: true become_user: "{{ cold_start_monitor_user }}" diff --git a/ops/monitoring/alerts-unified.yml b/ops/monitoring/alerts-unified.yml index 253873d5f..f30809f58 100644 --- a/ops/monitoring/alerts-unified.yml +++ b/ops/monitoring/alerts-unified.yml @@ -24,16 +24,16 @@ groups: - record: awoooi_recovery_core_ready expr: | sum without(result) ( - awoooi_cold_start_last_result{host="110",scope="110_120_121_188",result="green"} == bool 1 + awoooi_cold_start_last_result{host="110",scope="110_112_120_121_188",result="green"} == bool 1 ) * on(host,scope) ( - awoooi_cold_start_warn_gates{host="110",scope="110_120_121_188"} == bool 0 + awoooi_cold_start_warn_gates{host="110",scope="110_112_120_121_188"} == bool 0 ) * on(host,scope) ( - awoooi_cold_start_blocked_gates{host="110",scope="110_120_121_188"} == bool 0 + awoooi_cold_start_blocked_gates{host="110",scope="110_112_120_121_188"} == bool 0 ) * on(host,scope) ( - (time() - awoooi_cold_start_last_green_timestamp{host="110",scope="110_120_121_188"}) < bool 3600 + (time() - awoooi_cold_start_last_green_timestamp{host="110",scope="110_112_120_121_188"}) < bool 3600 ) - record: awoooi_recovery_dr_offsite_ready @@ -2326,7 +2326,7 @@ groups: runbook: "檢查 `/home/wooo/logs/prometheus-rule-drift-guard.log` 與 `/home/wooo/monitoring/alerts.yml.guard.bak.*`，找出誰覆寫了 active rules。" - alert: ColdStartMonitorMissing - expr: absent(awoooi_cold_start_monitor_up{host="110",scope="110_120_121_188"}) + expr: absent(awoooi_cold_start_monitor_up{host="110",scope="110_112_120_121_188"}) for: 15m labels: severity: warning @@ -2342,7 +2342,7 @@ groups: runbook: "執行 scripts/reboot-recovery/install-cold-start-monitor-110.sh，確認 /home/wooo/node_exporter_textfiles/cold_start_recovery.prom" - alert: ColdStartMonitorStale - expr: time() - awoooi_cold_start_last_run_timestamp{host="110",scope="110_120_121_188"} > 900 + expr: time() - awoooi_cold_start_last_run_timestamp{host="110",scope="110_112_120_121_188"} > 900 for: 10m labels: severity: warning @@ -2358,7 +2358,7 @@ groups: runbook: "SSH 110 檢查 crontab、/tmp/awoooi-cold-start-monitor.cron.log、cold-start-last.log" - alert: ColdStartRecoveryBlocked - expr: awoooi_cold_start_blocked_gates{host="110",scope="110_120_121_188"} > 0 + expr: awoooi_cold_start_blocked_gates{host="110",scope="110_112_120_121_188"} > 0 for: 5m labels: severity: critical @@ -2374,7 +2374,7 @@ groups: runbook: "查看 /home/wooo/reboot-recovery/cold-start-last.log；依 docs/runbooks/FULL-STACK-COLD-START-SOP.md 的 P0→P2 順序修復" - alert: K3sNodeFilesystemErrorGateBlocked - expr: awoooi_cold_start_blocker_reason{host="110",scope="110_120_121_188",reason="k3s_node_filesystem_error",target="120"} > 0 + expr: awoooi_cold_start_blocker_reason{host="110",scope="110_112_120_121_188",reason="k3s_node_filesystem_error",target="120"} > 0 for: 5m labels: severity: critical @@ -2392,7 +2392,7 @@ groups: runbook: "查看 110 `/home/wooo/reboot-recovery/cold-start-last.log`，執行 `scripts/reboot-recovery/120-fsck-maintenance-checklist.sh` 做維護前只讀檢查；維護窗內用 console/rescue 對 120 root LV 執行 fsck，禁止 online fsck。" - alert: ColdStartHost120Unreachable - expr: awoooi_cold_start_blocker_reason{host="110",scope="110_120_121_188",reason="host_unreachable",target="120"} > 0 + expr: awoooi_cold_start_blocker_reason{host="110",scope="110_112_120_121_188",reason="host_unreachable",target="120"} > 0 for: 3m labels: severity: critical @@ -2410,7 +2410,7 @@ groups: runbook: "查看 120 console。若停在 initramfs/manual fsck，先對 root LV 做離線 fsck；若主機關機或網卡異常，先恢復電源/網路，再重跑 full-stack cold-start gate。禁止從自動修復直接重啟其他服務掩蓋主機離線。" - alert: ColdStartRecoveryDegraded - expr: awoooi_cold_start_warn_gates{host="110",scope="110_120_121_188"} > 0 + expr: awoooi_cold_start_warn_gates{host="110",scope="110_112_120_121_188"} > 0 for: 15m labels: severity: warning @@ -2426,7 +2426,7 @@ groups: runbook: "查看 /home/wooo/reboot-recovery/cold-start-last.log，修到 PASS/WARN/BLOCKED = green" - alert: ColdStartLastGreenTooOld - expr: time() - awoooi_cold_start_last_green_timestamp{host="110",scope="110_120_121_188"} > 3600 + expr: time() - awoooi_cold_start_last_green_timestamp{host="110",scope="110_112_120_121_188"} > 3600 for: 15m labels: severity: warning @@ -2438,7 +2438,7 @@ groups: auto_repair: "false" annotations: summary: "Full-stack cold-start gate has not been GREEN recently" - description: "距離上次 GREEN 已超過 {{ $value | humanizeDuration }}；需要確認 110/120/121/188 與排程/網站 gate。" + description: "距離上次 GREEN 已超過 {{ $value | humanizeDuration }}；需要確認 110/112/120/121/188 與排程/網站 gate。" runbook: "執行 SSH_BATCH_MODE=yes bash scripts/reboot-recovery/full-stack-cold-start-check.sh --send-alert-test" # ========================================================================= diff --git a/ops/reboot-recovery/full-stack-cold-start-baseline.yml b/ops/reboot-recovery/full-stack-cold-start-baseline.yml index d0782f042..e1bde6fa3 100644 --- a/ops/reboot-recovery/full-stack-cold-start-baseline.yml +++ b/ops/reboot-recovery/full-stack-cold-start-baseline.yml @@ -351,7 +351,7 @@ authoritative_checks: expected: PASS=64 WARN=0 BLOCKED=0 textfile_metric: path: /home/wooo/node_exporter_textfiles/cold_start_recovery.prom - green_metric: awoooi_cold_start_last_result{host="110",scope="110_120_121_188",result="green"} 1 + green_metric: awoooi_cold_start_last_result{host="110",scope="110_112_120_121_188",result="green"} 1 backup_baseline: path: ops/reboot-recovery/full-stack-backup-baseline.yml required_metrics: diff --git a/scripts/ops/recovery-scorecard-contract-check.py b/scripts/ops/recovery-scorecard-contract-check.py index 5be1f88c1..7a9c98f6f 100755 --- a/scripts/ops/recovery-scorecard-contract-check.py +++ b/scripts/ops/recovery-scorecard-contract-check.py @@ -23,7 +23,7 @@ else: DEFAULT_RULES = Path("ops/monitoring/alerts-unified.yml") DEFAULT_BASELINE = Path("ops/reboot-recovery/full-stack-backup-baseline.yml") -EXPECTED_CORE = 'awoooi_recovery_core_ready{host="110",scope="110_120_121_188"}' +EXPECTED_CORE = 'awoooi_recovery_core_ready{host="110",scope="110_112_120_121_188"}' EXPECTED_DR = 'awoooi_recovery_dr_offsite_ready{host="110"}' diff --git a/scripts/reboot-recovery/full-stack-recovery-scorecard.sh b/scripts/reboot-recovery/full-stack-recovery-scorecard.sh index 223548373..27937e255 100755 --- a/scripts/reboot-recovery/full-stack-recovery-scorecard.sh +++ b/scripts/reboot-recovery/full-stack-recovery-scorecard.sh @@ -151,9 +151,9 @@ echo "AWOOOI full-stack recovery scorecard" date '+%Y-%m-%d %H:%M:%S %Z' echo -cold_green="$(bool_metric 'awoooi_cold_start_last_result{host="110",scope="110_120_121_188",result="green"}')" -cold_warn="$(metric_value 'awoooi_cold_start_warn_gates{host="110",scope="110_120_121_188"}' 2>/dev/null || echo 999)" -cold_blocked="$(metric_value 'awoooi_cold_start_blocked_gates{host="110",scope="110_120_121_188"}' 2>/dev/null || echo 999)" +cold_green="$(bool_metric 'awoooi_cold_start_last_result{host="110",scope="110_112_120_121_188",result="green"}')" +cold_warn="$(metric_value 'awoooi_cold_start_warn_gates{host="110",scope="110_112_120_121_188"}' 2>/dev/null || echo 999)" +cold_blocked="$(metric_value 'awoooi_cold_start_blocked_gates{host="110",scope="110_112_120_121_188"}' 2>/dev/null || echo 999)" cold_alerts="$(metric_value 'count(ALERTS{alertname=~"ColdStart.*",alertstate="firing"})' 2>/dev/null || echo 999)" status_value CORE_COLD_START_GREEN "$cold_green" diff --git a/scripts/reboot-recovery/tests/test_cold_start_monitor_bounded_probes.py b/scripts/reboot-recovery/tests/test_cold_start_monitor_bounded_probes.py index 05270aaa4..7e06ecab2 100644 --- a/scripts/reboot-recovery/tests/test_cold_start_monitor_bounded_probes.py +++ b/scripts/reboot-recovery/tests/test_cold_start_monitor_bounded_probes.py @@ -72,6 +72,14 @@ def test_full_stack_cold_start_includes_host112_as_a_required_p0_gate() -> None: encoding="utf-8" ) deploy_verifier = VERIFY_DEPLOY.read_text(encoding="utf-8") + alerts = (ROOT / "ops" / "monitoring" / "alerts-unified.yml").read_text( + encoding="utf-8" + ) + recovery_scorecard = RECOVERY_SCORECARD.read_text(encoding="utf-8") + ansible_verifier = ( + ROOT / "infra" / "ansible" / "roles" / "cold-start-monitor" + / "tasks" / "main.yml" + ).read_text(encoding="utf-8") assert "Scope: 110 / 112 / 120 / 121 / 188." in text assert "for host in 110 112 120 121 188; do" in text @@ -86,6 +94,15 @@ def test_full_stack_cold_start_includes_host112_as_a_required_p0_gate() -> None: assert 'reason="host_unreachable",target="112"' in exporter assert 'reason="guest_readiness_failed",target="112"' in exporter assert 'scope="110_112_120_121_188"' in deploy_verifier + assert 'base_url + "/api/v1/rules"' in deploy_verifier + assert "COLD_START_RULE_SCOPE_OK" in deploy_verifier + assert 'stale_scope = \'scope="110_120_121_188"\'' in deploy_verifier + assert 'scope="110_112_120_121_188"' in alerts + assert 'scope="110_112_120_121_188"' in recovery_scorecard + assert 'scope="110_112_120_121_188"' in ansible_verifier + assert 'scope="110_120_121_188"' not in alerts + assert 'scope="110_120_121_188"' not in recovery_scorecard + assert 'scope="110_120_121_188"' not in ansible_verifier def test_cold_start_momo_current_month_handles_no_new_source_without_false_warn() -> None: diff --git a/scripts/reboot-recovery/verify-cold-start-monitor-deploy.sh b/scripts/reboot-recovery/verify-cold-start-monitor-deploy.sh index c968df054..231d4dc9a 100755 --- a/scripts/reboot-recovery/verify-cold-start-monitor-deploy.sh +++ b/scripts/reboot-recovery/verify-cold-start-monitor-deploy.sh @@ -95,6 +95,63 @@ else: PY } +verify_cold_start_rule_scope() { + PROMETHEUS_URL="$PROMETHEUS_URL" python3 - <<'PY' +import json +import os +import urllib.request + +base_url = os.environ["PROMETHEUS_URL"].rstrip("/") +canonical_scope = 'scope="110_112_120_121_188"' +stale_scope = 'scope="110_120_121_188"' +required_rules = { + "awoooi_recovery_core_ready", + "ColdStartMonitorMissing", + "ColdStartMonitorStale", + "ColdStartRecoveryBlocked", + "K3sNodeFilesystemErrorGateBlocked", + "ColdStartHost120Unreachable", + "ColdStartRecoveryDegraded", + "ColdStartLastGreenTooOld", +} + +try: + payload = json.load( + urllib.request.urlopen(base_url + "/api/v1/rules", timeout=8) + ) +except Exception as exc: # noqa: BLE001 + raise SystemExit(f"BLOCKED cold-start rule scope readback failed: {exc}") from exc + +if payload.get("status") != "success": + raise SystemExit("BLOCKED cold-start rule scope readback returned non-success") + +queries = {} +for group in payload.get("data", {}).get("groups", []): + for rule in group.get("rules", []): + name = str(rule.get("name") or "") + if name in required_rules: + queries[name] = str(rule.get("query") or rule.get("expr") or "") + +missing = sorted(required_rules - queries.keys()) +wrong_scope = sorted( + name for name, query in queries.items() if canonical_scope not in query +) +stale = sorted(name for name, query in queries.items() if stale_scope in query) +if missing or wrong_scope or stale: + raise SystemExit( + "BLOCKED cold-start rule scope drift " + f"missing={','.join(missing) or 'none'} " + f"wrong_scope={','.join(wrong_scope) or 'none'} " + f"stale_scope={','.join(stale) or 'none'}" + ) + +print( + "COLD_START_RULE_SCOPE_OK " + f"scope=110_112_120_121_188 rules={len(required_rules)} stale_scope=0" +) +PY +} + report_runtime_state() { remote_read "awk ' /awoooi_cold_start_monitor_up/ {monitor_up=\$NF} @@ -139,6 +196,7 @@ require_remote_pattern \ "110 cold-start monitor exports host112 readiness blocker series" report_runtime_state +verify_cold_start_rule_scope report_cold_start_alerts echo "COLD_START_MONITOR_DEPLOY_PARITY_OK"