From 8dbbef80d4347f957ad5885fc67b1d273a0a8e77 Mon Sep 17 00:00:00 2001 From: Your Name Date: Thu, 2 Jul 2026 21:20:29 +0800 Subject: [PATCH] feat(reboot): verify public maintenance fallback --- docs/LOGBOOK.md | 18 ++ .../PUBLIC-MAINTENANCE-FALLBACK-RUNBOOK.md | 19 ++ ...r-inserted-requirements-priority-ledger.md | 6 +- .../public-maintenance-fallback-probe.py | 213 ++++++++++++++++++ .../reboot-auto-recovery-slo-exporter.sh | 48 ++++ .../reboot-auto-recovery-slo-scorecard.py | 173 ++++++++++++++ .../test_reboot_p0_operational_contract.py | 127 +++++++++++ 7 files changed, 601 insertions(+), 3 deletions(-) create mode 100644 scripts/reboot-recovery/public-maintenance-fallback-probe.py diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index 8de51c5cd..b8dfbf6a3 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -1,3 +1,21 @@ +## 2026-07-02 — 21:24 P0-006 public maintenance fallback runtime gate + +**完成內容**： +- 接續全主機重啟 P0 主線，先將本機工作樹 fast-forward 到 Gitea deploy marker `2ba12ca59`，對齊已部署 source `8982fc5bddc561d4a51ec41e1456972eb0a74a9d`。 +- Gitea CD `#4446` build/deploy log 顯示 tests `602 passed`、API / Web / Worker rollout success、production desired image tag 與 runtime build commit 均 matched `8982fc5bdd`；public `/api/v1/health` 已回 `200`。 +- 同一輪 rollout 曾讀回 public `502` / `public_health_argocd_wait_http=502`，證明 source 有 `maintenance.html` 仍不足以保證使用者不會看到 raw 502。 +- 新增 `scripts/reboot-recovery/public-maintenance-fallback-probe.py`：只讀 public routes，輸出 raw 5xx without fallback、L0 `X-AWOOOI-Fallback=local-maintenance`、L1 unreachable without external fallback 的 JSON / metric。 +- `reboot-auto-recovery-slo-exporter.sh` 已把 public fallback probe artifact 傳入 `reboot-auto-recovery-slo-scorecard.py`；scorecard 新增 `public_maintenance_fallback_runtime_ready` required check、`public_route_raw_5xx_without_maintenance_fallback` / `public_route_unreachable_without_external_l1_fallback` / `public_maintenance_fallback_runtime_readback_missing` blockers 與 rollups。 +- `PUBLIC-MAINTENANCE-FALLBACK-RUNBOOK.md` 與 `2026-07-02-commander-inserted-requirements-priority-ledger.md` 已同步：verifier 已完成，但 L0 Nginx 實際 intercept 與 L1 external/CDN fallback 仍需 production/edge readback，不可宣稱完成。 + +**驗證待跑**： +- focused pytest：`scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py`。 +- syntax / guard：`python3 -m py_compile scripts/reboot-recovery/public-maintenance-fallback-probe.py scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py`、`bash -n scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh`、`python3 ops/runner/guard-gitea-runner-pressure.py --root .`。 +- production probe：跑 `public-maintenance-fallback-probe.py` 讀回目前 public routes；若 raw 502 再推 L0 Nginx staging verifier，若 edge unreachable 再推 L1 external static fallback decision record。 + +**仍維持**： +- 未使用 GitHub / `gh` / GitHub API；未讀 secret / token / `.env` / raw sessions / SQLite / auth；未 workflow_dispatch；未重啟主機 / VM / Docker / Nginx / K3s / DB / firewall；未切 DNS / CDN / provider；未宣稱 10 分鐘全主機自動恢復 SLA 已完成。 + ## 2026-07-02 — 20:56 CIR-P0-TG-001 Telegram alert AI Loop post-apply verifier readback **完成內容**： diff --git a/docs/runbooks/PUBLIC-MAINTENANCE-FALLBACK-RUNBOOK.md b/docs/runbooks/PUBLIC-MAINTENANCE-FALLBACK-RUNBOOK.md index 450e43bf5..e62f80744 100644 --- a/docs/runbooks/PUBLIC-MAINTENANCE-FALLBACK-RUNBOOK.md +++ b/docs/runbooks/PUBLIC-MAINTENANCE-FALLBACK-RUNBOOK.md @@ -40,3 +40,22 @@ L1 active 條件： - L0：對測試 vhost 讀回 `200` 或 `503` 的維護頁 HTML，且 header `X-AWOOOI-Fallback=local-maintenance` 存在。 - L1：從外部網路讀回靜態維護頁，且不經過 99 / 110 / 188 / 120 / 121 / 112。 - Recovery：主要 upstream 連續 2 分鐘健康後切回正式 origin；Telegram 發送 recovery 通知，並在 cold-start scorecard 留下 readback。 + +## 2026-07-02 scorecard gate + +2026-07-02 晚間 Gitea CD `#4446` rollout 期間曾讀回 public `502`，後續正式 health 回到 `200`。這證明「source 有維護頁」不足以代表使用者不會看到 raw 502；驗收必須要求 runtime readback。 + +固定驗證命令： + +```bash +python3 scripts/reboot-recovery/public-maintenance-fallback-probe.py \ + --output /tmp/public-maintenance-fallback.json +python3 scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py \ + --public-maintenance-file /tmp/public-maintenance-fallback.json +``` + +判斷規則： + +- `public_route_raw_5xx_without_maintenance_fallback`：L0 Nginx intercept 未生效，下一步只能做 staging vhost / `nginx -t` / header verifier，不得用 production app restart 當測試。 +- `public_route_unreachable_without_external_l1_fallback`：edge 或外部路徑整段不可達，下一步是 L1 external static origin / CDN health check decision record 與 rollback，不直接切 DNS。 +- `public_maintenance_fallback_runtime_readback_missing`：沒有 runtime probe 證據，不得宣稱 public maintenance fallback 已完成。 diff --git a/docs/workplans/2026-07-02-commander-inserted-requirements-priority-ledger.md b/docs/workplans/2026-07-02-commander-inserted-requirements-priority-ledger.md index 3de272fad..47ff74142 100644 --- a/docs/workplans/2026-07-02-commander-inserted-requirements-priority-ledger.md +++ b/docs/workplans/2026-07-02-commander-inserted-requirements-priority-ledger.md @@ -61,7 +61,7 @@ | 2 | CIR-P0-RBT-002 | P0 | 「沒有偵測到主機重啟」 | 修正 host reboot/shutdown/up detection：boot_id / uptime / node exporter / Windows exporter / VMware VM power state 都要進同一事件 | Scorecard 已接 collection packet + management probe；99 host reachable 但 uptime unknown，111 unreachable，stale hosts 仍存在 | 讓 99 verifier / Windows exporter 或等效 no-secret readback 進入 host boot event，並補 111 reachability 證據 | | 3 | CIR-P0-RBT-003 | P0 | 「192.168.0.99 VMWare 要自動啟動，裡面 111/188/120/121/112 也自動啟動」 | Windows 99 VMware host autostart + guest VM autostart contract；VM host 111/188/120/121/112 開機順序與 readback | Source verifier / parser / API readback / collection packet 已完成；management probe 讀回 `host_reachable=true`、RDP open、`2179` VMConnect / console channel open、SSH BatchMode `permission_denied`、WinRM timeout；snapshot active blockers=`windows99_remote_execution_channel_unavailable`、`windows99_vmware_autostart_readback_missing` | 收集 local console Verify output 或恢復 no-secret management channel，再確認 `VMRUN_PRESENT`、scheduled task、VMware services、VM power、VMX present 全綠 | | 4 | CIR-P0-RBT-004 | P0 | 「192.168.0.99 不可因 Windows Update 無預警重開」 | Windows Update reboot policy：active hours / no auto-restart / maintenance window / update notification audit | Source verifier 已補 `WINDOWS_UPDATE_POLICY` 與 `WINDOWS_UPDATE_NO_AUTO_REBOOT_READY`；collection packet 已列 forbidden actions；99 management channel 尚不能收 policy readback | 取得 Verify output；若 policy 不綠，再走 controlled apply，禁止要求或記錄 Windows 密碼 | -| 5 | CIR-P0-RBT-005 | P0 | 「網站重啟後 502 嚴重影響體驗，要維護頁，外部雲端或專業做法」 | Public maintenance fallback：Nginx / edge / external static maintenance page / status page / fail-open UX，避免 502 直出 | 尚未完整落地；目前是需求缺口 | 產生 `public_maintenance_fallback` decision record：DNS/edge/外部雲端/本地 Nginx fallback 風險比較，先做不切流量的 check-mode | +| 5 | CIR-P0-RBT-005 | P0 | 「網站重啟後 502 嚴重影響體驗，要維護頁，外部雲端或專業做法」 | Public maintenance fallback：Nginx / edge / external static maintenance page / status page / fail-open UX，避免 502 直出 | Source + runtime verifier 已實作：`public-maintenance-fallback-probe.py` 會抓 raw 5xx / L0 header / L1 unreachable，scorecard 會 fail-closed；L0 Nginx 實際套用與 L1 external/CDN cutover 尚需 runtime readback | 先讀回 production probe；若出現 `public_route_raw_5xx_without_maintenance_fallback`，只做 staging vhost / `nginx -t` / header verifier；若 edge unreachable，產生 L1 external static origin / CDN decision record + rollback | | 6 | CIR-P0-RBT-006 | P0 | 「所有主機關機立刻 Telegram 告警，重啟後也要告警，其他告警一併完整思考」 | Down / shutdown suspected / reboot detected / reboot recovered / SLO missed / backup failed / freshness stale / CPU pressure / Gitea queue 告警矩陣 | 部分已有 Alertmanager rule 與 Telegram receipt 補強；仍缺完整 shutdown/up E2E receipt | 建立 Telegram alert matrix + receipt verifier，逐項讀回 Alertmanager active/resolved 與 outbound receipt，不送測試 secret | | 7 | CIR-P0-RBT-007 | P0 | 「所有備份包含主機、DB、網站、服務、套件、工具、日誌都沒有監控告警」 | Backup observability coverage：backup job inventory、last success、freshness、offsite、restore drill、Telegram receipt | 部分已有 backup health exporter / alert rules；全域 coverage 與 restore drill 未全綠 | 建立 backup coverage matrix：host / DB / website / service config / package list / tool scripts / logs，每列有 metric、alert、last_success、restore_verifier | | 8 | CIR-P0-RBT-008 | P0 | 「每次重啟排查都不一樣，也不知道多久恢復，不符合 SLA」 | 固定化 reboot runbook：fixed triage order、ETA、active blocker、remaining seconds、owner lane、next command | Scorecard/API contract 已固定 `next_safe_action=restore_windows99_no_secret_management_channel_or_collect_local_console_verify_readback_then_rerun_reboot_scorecard_no_reboot`；尚待 deploy readback | 推送後讀回 `/api/v1/agents/reboot-auto-recovery-slo-scorecard`，確認 production 顯示 99 management-channel blocker 與 readiness `43%` | @@ -132,7 +132,7 @@ | AI 專業 UI / 非文字牆 cockpit | 已列為 P2 UX 驗收 | 將長文字區塊收斂成 first-viewport cockpit、cards、flow rows 與 expandable details | | 10 分鐘 reboot auto-recovery SLA | scorecard source / snapshot / API 已補固定 phase / ETA / blocker / next action 欄位；仍缺 fresh all-host reboot/drill proof | 補 production API readback，缺事件則明確標等待下一次 reboot 或 approved drill | | 99 Windows / VMware autostart | Source verifier / parser / API readback / collection packet 已完成；live 99 Verify output 尚未收集，scorecard 以 `windows99_vmware_autostart_readback_missing` fail-closed；collection packet 顯示 99 reachable、可收 no-secret Verify，但 uptime unknown | 收集 99 no-secret Verify output，確認 VM 111/188/120/121/112 running、scheduled task / services / Windows Update policy 全綠 | -| 502 maintenance fallback | 尚未完成外部維護頁 / edge fallback 決策與實作 | 先做 no-write decision record + smoke verifier | +| 502 maintenance fallback | Source + scorecard verifier 已補；真實 L0/L1 runtime 套用仍待 probe / edge readback，不可宣稱完成 | 先跑 production probe；raw 5xx 則推 L0 Nginx intercept，edge unreachable 則推 L1 external static fallback decision + rollback | | 全備份監控告警 coverage | 部分 exporter/rule 已存在，但 host/DB/site/service/package/tool/log coverage 未全列 | 建立 backup coverage matrix 與 restore drill verifier | | Stock/Postgres hot pressure | 110 live 已導向 Stock/Postgres playbook；尚未完成 hot query / backup export playbook closure | 下一步執行 read-only Stock/Postgres evidence 與 source freshness / query attribution | @@ -153,7 +153,7 @@ 3. 接續目前 active P0：110 Stock/Postgres hot pressure，跑 `postgres_hot_query_or_backup_export_playbook` 的 read-only evidence / source freshness / query attribution。 4. 補全 reboot auto-recovery P0：99/110/111/112/120/121/188 reboot detection、10 分鐘 SLO scorecard、Telegram down/up/recovered/SLO missed receipts。 5. 補 99 Windows / VMware autostart P0：99 host 自動啟動 VMware，VM 111/188/120/121/112 自動啟動；同時補 Windows Update no-auto-restart verifier。 -6. 補 public maintenance fallback P0：避免 502 直出，先完成 external/edge/local fallback decision record 與 no-write smoke。 +6. 補 public maintenance fallback P0：先用 `public-maintenance-fallback-probe.py` / scorecard 讀回 production；raw 502 走 L0 Nginx intercept verifier，edge unreachable 走 L1 external static fallback decision + rollback。 7. 補 backup observability P0：host / DB / website / service config / package list / tools / logs backup matrix、last success、offsite、restore drill、Telegram receipt。 8. 補 product freshness/version P0：所有產品網站 source SHA / deploy marker / runtime image / public URL / data freshness readback。 9. 繼續補 110 controlled drain lane 的 staging / verifier，不恢復 generic runner。 diff --git a/scripts/reboot-recovery/public-maintenance-fallback-probe.py b/scripts/reboot-recovery/public-maintenance-fallback-probe.py new file mode 100644 index 000000000..6b3fb7ca4 --- /dev/null +++ b/scripts/reboot-recovery/public-maintenance-fallback-probe.py @@ -0,0 +1,213 @@ +#!/usr/bin/env python3 +"""Probe public routes for raw 5xx exposure without a maintenance fallback.""" + +from __future__ import annotations + +import argparse +import json +import socket +import sys +import urllib.error +import urllib.request +from datetime import datetime +from pathlib import Path +from typing import Any + + +SCHEMA_VERSION = "awoooi_public_maintenance_fallback_probe_v1" +DEFAULT_URLS = [ + "https://awoooi.wooo.work/api/v1/health", + "https://awoooi.wooo.work/", + "https://stock.wooo.work/api/v1/system/freshness", + "https://mo.wooo.work/health", + "https://bitan.wooo.work/", + "https://www.tsenyang.com/", +] +MAINTENANCE_MARKERS = [ + "AWOOOI 維護模式", + "系統正在恢復服務", + "HTTP 502 / 503 / 504 fallback page", +] + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Read-only public route verifier for maintenance fallback.", + ) + parser.add_argument( + "--url", + action="append", + dest="urls", + help="Public URL to probe. May be passed more than once.", + ) + parser.add_argument("--timeout", type=float, default=8.0) + parser.add_argument("--max-bytes", type=int, default=65536) + parser.add_argument("--generated-at", help="Override generated_at for snapshots.") + parser.add_argument("--output", type=Path, help="Write JSON report to this path.") + parser.add_argument( + "--prometheus-output", + type=Path, + help="Optionally write node_exporter textfile metrics.", + ) + return parser.parse_args() + + +def fetch_url(url: str, *, timeout: float, max_bytes: int) -> dict[str, Any]: + request = urllib.request.Request( + url, + headers={ + "User-Agent": "awoooi-reboot-public-maintenance-probe/1.0", + "Accept": "text/html,application/json;q=0.9,*/*;q=0.1", + }, + ) + status = 0 + headers: dict[str, str] = {} + body = b"" + error = "" + try: + with urllib.request.urlopen(request, timeout=timeout) as response: + status = int(response.getcode() or 0) + headers = {key.lower(): value for key, value in response.headers.items()} + body = response.read(max_bytes) + except urllib.error.HTTPError as exc: + status = int(exc.code or 0) + headers = {key.lower(): value for key, value in exc.headers.items()} + try: + body = exc.read(max_bytes) + except OSError: + body = b"" + except (urllib.error.URLError, TimeoutError, socket.timeout) as exc: + error = str(getattr(exc, "reason", exc)) + + text = body.decode("utf-8", errors="replace") + fallback_header = headers.get("x-awoooi-fallback", "") + maintenance_body_detected = any(marker in text for marker in MAINTENANCE_MARKERS) + maintenance_fallback_serving = ( + status in {200, 503} + and fallback_header == "local-maintenance" + and maintenance_body_detected + ) + raw_5xx_without_fallback = ( + status >= 500 + and not maintenance_fallback_serving + ) + route_unreachable_without_external_fallback = status == 0 + if maintenance_fallback_serving: + route_status = "maintenance_fallback_serving" + elif raw_5xx_without_fallback: + route_status = "raw_5xx_without_maintenance_fallback" + elif route_unreachable_without_external_fallback: + route_status = "route_unreachable_without_external_fallback" + elif 200 <= status < 500: + route_status = "normal_origin_serving" + else: + route_status = "unknown" + + return { + "url": url, + "http_status": status, + "status": route_status, + "fallback_header": fallback_header, + "maintenance_body_detected": maintenance_body_detected, + "maintenance_fallback_serving": maintenance_fallback_serving, + "raw_5xx_without_fallback": raw_5xx_without_fallback, + "route_unreachable_without_external_fallback": ( + route_unreachable_without_external_fallback + ), + "error": error[:300], + } + + +def build_report(args: argparse.Namespace) -> dict[str, Any]: + generated_at = args.generated_at or datetime.now().astimezone().isoformat( + timespec="seconds" + ) + urls = args.urls or DEFAULT_URLS + routes = [ + fetch_url(url, timeout=args.timeout, max_bytes=args.max_bytes) for url in urls + ] + raw_5xx_without_fallback = [ + row["url"] for row in routes if row["raw_5xx_without_fallback"] + ] + unreachable_without_external_fallback = [ + row["url"] + for row in routes + if row["route_unreachable_without_external_fallback"] + ] + maintenance_fallback_routes = [ + row["url"] for row in routes if row["maintenance_fallback_serving"] + ] + blockers: list[str] = [] + if raw_5xx_without_fallback: + blockers.append("public_route_raw_5xx_without_maintenance_fallback") + if unreachable_without_external_fallback: + blockers.append("public_route_unreachable_without_external_l1_fallback") + return { + "schema_version": SCHEMA_VERSION, + "generated_at": generated_at, + "runtime_readback_present": True, + "ready": not blockers, + "target_url_count": len(urls), + "route_count": len(routes), + "raw_5xx_without_fallback_count": len(raw_5xx_without_fallback), + "route_unreachable_without_external_fallback_count": len( + unreachable_without_external_fallback + ), + "maintenance_fallback_route_count": len(maintenance_fallback_routes), + "raw_5xx_without_fallback_urls": raw_5xx_without_fallback, + "route_unreachable_without_external_fallback_urls": ( + unreachable_without_external_fallback + ), + "maintenance_fallback_urls": maintenance_fallback_routes, + "blockers": blockers, + "routes": routes, + "forbidden_actions": [ + "nginx_reload_from_probe", + "dns_cutover_from_probe", + "service_restart_from_probe", + "host_reboot_from_probe", + "secret_value_read", + ], + } + + +def write_prometheus(path: Path, report: dict[str, Any]) -> None: + ready = 1 if report.get("ready") is True else 0 + raw_5xx = int(report.get("raw_5xx_without_fallback_count") or 0) + unreachable = int( + report.get("route_unreachable_without_external_fallback_count") or 0 + ) + text = "\n".join( + [ + "# HELP awoooi_public_maintenance_fallback_ready Whether public routes avoid raw 5xx exposure.", + "# TYPE awoooi_public_maintenance_fallback_ready gauge", + f'awoooi_public_maintenance_fallback_ready{{scope="public_routes"}} {ready}', + "# HELP awoooi_public_route_raw_5xx_without_maintenance_fallback Raw public 5xx responses without maintenance fallback.", + "# TYPE awoooi_public_route_raw_5xx_without_maintenance_fallback gauge", + f'awoooi_public_route_raw_5xx_without_maintenance_fallback{{scope="public_routes"}} {raw_5xx}', + "# HELP awoooi_public_route_unreachable_without_external_l1_fallback Public routes unreachable without external fallback proof.", + "# TYPE awoooi_public_route_unreachable_without_external_l1_fallback gauge", + f'awoooi_public_route_unreachable_without_external_l1_fallback{{scope="public_routes"}} {unreachable}', + "", + ] + ) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + + +def main() -> int: + args = parse_args() + report = build_report(args) + text = json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) + "\n" + if args.output: + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(text, encoding="utf-8") + else: + print(text, end="") + if args.prometheus_output: + write_prometheus(args.prometheus_output, report) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh b/scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh index e5c9923b2..cd1e2066c 100755 --- a/scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh +++ b/scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh @@ -16,6 +16,8 @@ LOCK_FILE="${LOCK_FILE:-${LOG_DIR}/reboot_auto_recovery_slo.lock}" STOCK_FRESHNESS_URL="${STOCK_FRESHNESS_URL:-https://stock.wooo.work/api/v1/system/freshness}" STOCK_INGESTION_URL="${STOCK_INGESTION_URL:-https://stock.wooo.work/api/v1/system/ingestion}" STOCK_READBACK_TIMEOUT_SECONDS="${STOCK_READBACK_TIMEOUT_SECONDS:-10}" +PUBLIC_MAINTENANCE_READBACK_TIMEOUT_SECONDS="${PUBLIC_MAINTENANCE_READBACK_TIMEOUT_SECONDS:-8}" +PUBLIC_MAINTENANCE_URLS="${PUBLIC_MAINTENANCE_URLS:-https://awoooi.wooo.work/api/v1/health https://awoooi.wooo.work/ https://stock.wooo.work/api/v1/system/freshness https://mo.wooo.work/health https://bitan.wooo.work/ https://www.tsenyang.com/}" mkdir -p "$TEXTFILE_DIR" "$LOG_DIR" @@ -35,6 +37,8 @@ summary_file="$artifact_dir/summary.txt" scorecard_file="$artifact_dir/scorecard.json" stock_freshness_file="$artifact_dir/stock-freshness.json" stock_ingestion_file="$artifact_dir/stock-ingestion.json" +public_maintenance_file="$artifact_dir/public-maintenance-fallback.json" +public_maintenance_prom="$artifact_dir/public-maintenance-fallback.prom" windows99_management_file="$artifact_dir/windows99-management-channel.json" reboot_event_state_file="${REBOOT_EVENT_STATE_FILE:-${LOG_DIR}/reboot-event-state.json}" @@ -57,6 +61,20 @@ if command -v curl >/dev/null 2>&1; then || rm -f "$stock_ingestion_file" fi +if [ -f "$ROOT_DIR/scripts/reboot-recovery/public-maintenance-fallback-probe.py" ]; then + public_probe_args=( + "$ROOT_DIR/scripts/reboot-recovery/public-maintenance-fallback-probe.py" + --timeout "$PUBLIC_MAINTENANCE_READBACK_TIMEOUT_SECONDS" + --output "$public_maintenance_file" + --prometheus-output "$public_maintenance_prom" + ) + for url in $PUBLIC_MAINTENANCE_URLS; do + public_probe_args+=(--url "$url") + done + python3 "${public_probe_args[@]}" >"$artifact_dir/public-maintenance-fallback.stdout" 2>"$artifact_dir/public-maintenance-fallback.err" \ + || rm -f "$public_maintenance_file" "$public_maintenance_prom" +fi + if [ -f "$ROOT_DIR/scripts/reboot-recovery/windows99-management-channel-probe.py" ]; then python3 "$ROOT_DIR/scripts/reboot-recovery/windows99-management-channel-probe.py" \ --output "$windows99_management_file" >"$artifact_dir/windows99-management-channel.stdout" 2>"$artifact_dir/windows99-management-channel.err" \ @@ -79,6 +97,9 @@ fi if [ -s "$stock_ingestion_file" ]; then scorecard_args+=(--stock-ingestion-file "$stock_ingestion_file") fi +if [ -s "$public_maintenance_file" ]; then + scorecard_args+=(--public-maintenance-file "$public_maintenance_file") +fi if [ -s "$windows99_management_file" ]; then scorecard_args+=(--windows99-management-file "$windows99_management_file") fi @@ -110,6 +131,24 @@ payload=json.load(open(sys.argv[1], encoding="utf-8")) print(payload.get("sla_recovery_eta", {}).get("target_seconds_remaining", 0) or 0) PY )" +public_maintenance_ready="$(python3 - "$scorecard_file" <<'PY' +import json, sys +payload=json.load(open(sys.argv[1], encoding="utf-8")) +print(1 if payload.get("public_maintenance_fallback", {}).get("ready") else 0) +PY +)" +public_raw_5xx_count="$(python3 - "$scorecard_file" <<'PY' +import json, sys +payload=json.load(open(sys.argv[1], encoding="utf-8")) +print(payload.get("public_maintenance_fallback", {}).get("raw_5xx_without_fallback_count", 0) or 0) +PY +)" +public_unreachable_count="$(python3 - "$scorecard_file" <<'PY' +import json, sys +payload=json.load(open(sys.argv[1], encoding="utf-8")) +print(payload.get("public_maintenance_fallback", {}).get("route_unreachable_without_external_fallback_count", 0) or 0) +PY +)" tmp_metric="$(mktemp "$TEXTFILE_DIR/.reboot_auto_recovery_slo.XXXXXX")" cat >"$tmp_metric" <>"$tmp_metric" diff --git a/scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py b/scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py index bf8ccb25c..a49318fc1 100755 --- a/scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py +++ b/scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py @@ -44,6 +44,11 @@ def parse_args() -> argparse.Namespace: type=Path, help="Optional host pressure JSON readback from Prometheus / node exporter.", ) + parser.add_argument( + "--public-maintenance-file", + type=Path, + help="Optional public maintenance fallback route probe JSON readback.", + ) parser.add_argument( "--windows99-vmware-file", type=Path, @@ -347,6 +352,93 @@ def read_json_object(path: Path | None) -> dict[str, Any]: return payload +def parse_public_maintenance_readback(path: Path | None) -> dict[str, Any]: + """Parse public route maintenance fallback probe JSON.""" + default = { + "runtime_readback_present": False, + "ready": False, + "target_url_count": 0, + "route_count": 0, + "raw_5xx_without_fallback_count": 0, + "route_unreachable_without_external_fallback_count": 0, + "maintenance_fallback_route_count": 0, + "raw_5xx_without_fallback_urls": [], + "route_unreachable_without_external_fallback_urls": [], + "maintenance_fallback_urls": [], + "routes": [], + "blockers": ["public_maintenance_fallback_runtime_readback_missing"], + } + payload = read_json_object(path) + if not payload: + return default + + routes = payload.get("routes") + if not isinstance(routes, list): + routes = [] + compact_routes: list[dict[str, Any]] = [] + for item in routes: + if not isinstance(item, dict): + continue + compact_routes.append( + { + "url": str(item.get("url") or ""), + "http_status": int_value(item.get("http_status")), + "status": str(item.get("status") or "unknown"), + "fallback_header": str(item.get("fallback_header") or ""), + "maintenance_body_detected": ( + item.get("maintenance_body_detected") is True + ), + "maintenance_fallback_serving": ( + item.get("maintenance_fallback_serving") is True + ), + "raw_5xx_without_fallback": ( + item.get("raw_5xx_without_fallback") is True + ), + "route_unreachable_without_external_fallback": ( + item.get("route_unreachable_without_external_fallback") is True + ), + "error": str(item.get("error") or "")[:300], + } + ) + + blockers = strings(payload.get("blockers")) + raw_5xx_count = int_value(payload.get("raw_5xx_without_fallback_count")) + unreachable_count = int_value( + payload.get("route_unreachable_without_external_fallback_count") + ) + if raw_5xx_count and "public_route_raw_5xx_without_maintenance_fallback" not in blockers: + blockers.append("public_route_raw_5xx_without_maintenance_fallback") + if ( + unreachable_count + and "public_route_unreachable_without_external_l1_fallback" not in blockers + ): + blockers.append("public_route_unreachable_without_external_l1_fallback") + + return { + "runtime_readback_present": True, + "schema_version": str(payload.get("schema_version") or "unknown"), + "generated_at": str(payload.get("generated_at") or ""), + "ready": payload.get("ready") is True and not blockers, + "target_url_count": int_value(payload.get("target_url_count")), + "route_count": int_value(payload.get("route_count")) or len(compact_routes), + "raw_5xx_without_fallback_count": raw_5xx_count, + "route_unreachable_without_external_fallback_count": unreachable_count, + "maintenance_fallback_route_count": int_value( + payload.get("maintenance_fallback_route_count") + ), + "raw_5xx_without_fallback_urls": strings( + payload.get("raw_5xx_without_fallback_urls") + ), + "route_unreachable_without_external_fallback_urls": strings( + payload.get("route_unreachable_without_external_fallback_urls") + ), + "maintenance_fallback_urls": strings(payload.get("maintenance_fallback_urls")), + "routes": compact_routes, + "blockers": blockers, + "forbidden_actions": strings(payload.get("forbidden_actions")), + } + + def parse_host_probe(text: str) -> list[dict[str, Any]]: rows: list[dict[str, Any]] = [] for raw_line in text.splitlines(): @@ -446,6 +538,12 @@ def source_controls() -> dict[str, bool]: "public_maintenance_fallback_source_present": source_file( "ops/maintenance/maintenance.html" ).exists() + and file_contains( + source_file("ops/maintenance/nginx-502-maintenance-snippet.conf"), + "proxy_intercept_errors on", + "error_page 502 503 504", + "X-AWOOOI-Fallback", + ) and file_contains( source_file("docs/runbooks/PUBLIC-MAINTENANCE-FALLBACK-RUNBOOK.md"), "L0", @@ -886,6 +984,7 @@ def choose_safe_next_step( blockers: list[str], stockplatform: dict[str, Any], host_pressure: dict[str, Any], + public_maintenance: dict[str, Any], ) -> str: freshness_status = str(stockplatform.get("freshness_status") or "unknown") eod_window = stockplatform.get("eod_window") if isinstance(stockplatform.get("eod_window"), dict) else {} @@ -934,6 +1033,22 @@ def choose_safe_next_step( "collect_windows99_vmware_autostart_verify_readback_then_rerun_all_host_" "reboot_scorecard_no_secret_no_reboot" ) + public_blockers = set(strings(public_maintenance.get("blockers"))) + if "public_route_raw_5xx_without_maintenance_fallback" in public_blockers: + return ( + "apply_public_maintenance_fallback_l0_nginx_intercept_with_nginx_t_" + "test_staging_vhost_header_verifier_then_deploy_no_app_restart" + ) + if "public_route_unreachable_without_external_l1_fallback" in public_blockers: + return ( + "prepare_public_maintenance_fallback_l1_external_static_origin_or_cdn_" + "healthcheck_decision_record_then_controlled_cutover_with_rollback" + ) + if "public_maintenance_fallback_runtime_readback_missing" in blockers: + return ( + "run_public_maintenance_fallback_probe_and_rerun_reboot_scorecard_" + "verify_only_no_nginx_reload" + ) if blockers == ["host_boot_observation_older_than_target_window"]: return ( "timer_deployed_and_services_readback_green_wait_for_next_all_host_reboot_" @@ -963,6 +1078,9 @@ def build_required_checks(payload: dict[str, Any]) -> dict[str, bool]: stockplatform = payload.get("stockplatform_data_freshness") if not isinstance(stockplatform, dict): stockplatform = {} + public_maintenance = payload.get("public_maintenance_fallback") + if not isinstance(public_maintenance, dict): + public_maintenance = {} windows99 = payload.get("windows99_vmware_autostart") if not isinstance(windows99, dict): windows99 = {} @@ -1001,6 +1119,9 @@ def build_required_checks(payload: dict[str, Any]) -> dict[str, bool]: "stockplatform_ingestion_ok": ( stockplatform.get("ingestion_status") in {"ok", "unknown"} ), + "public_maintenance_fallback_runtime_ready": ( + public_maintenance.get("ready") is True + ), "fresh_reboot_window_observed": not strings( host_boot_detection.get("stale_hosts") ), @@ -1042,6 +1163,8 @@ def reboot_sop_current_phase(active_blockers: list[str], can_claim: bool) -> str "host_188_service_green_not_1", "wazuh_dashboard_degraded", } + if any(blocker.startswith("public_") for blocker in active_blockers): + return "public_maintenance_fallback_blocked" if any(blocker in service_blockers for blocker in active_blockers): return "post_reboot_service_readiness_blocked" if any("stockplatform" in blocker or "product_data" in blocker for blocker in active_blockers): @@ -1072,6 +1195,9 @@ def reboot_sop_primary_blocker(active_blockers: list[str]) -> str: "windows99_vmware_autostart_config_not_ready", "windows99_vmware_guest_power_not_ready", "windows99_update_no_auto_reboot_policy_not_ready", + "public_route_raw_5xx_without_maintenance_fallback", + "public_route_unreachable_without_external_l1_fallback", + "public_maintenance_fallback_runtime_readback_missing", "post_start_blocked_not_zero", "service_green_not_1", "host_188_service_green_not_1", @@ -1159,6 +1285,9 @@ def enrich_machine_readback(payload: dict[str, Any]) -> dict[str, Any]: stockplatform = payload.get("stockplatform_data_freshness") if not isinstance(stockplatform, dict): stockplatform = {} + public_maintenance = payload.get("public_maintenance_fallback") + if not isinstance(public_maintenance, dict): + public_maintenance = {} windows99 = payload.get("windows99_vmware_autostart") if not isinstance(windows99, dict): windows99 = {} @@ -1253,6 +1382,23 @@ def enrich_machine_readback(payload: dict[str, Any]) -> dict[str, Any]: "stockplatform_ingestion_blocker_count": len( strings(stockplatform.get("ingestion_blockers")) ), + "public_maintenance_runtime_readback_present": ( + public_maintenance.get("runtime_readback_present") is True + ), + "public_maintenance_fallback_ready": ( + public_maintenance.get("ready") is True + ), + "public_route_raw_5xx_without_fallback_count": int_value( + public_maintenance.get("raw_5xx_without_fallback_count") + ), + "public_route_unreachable_without_external_fallback_count": int_value( + public_maintenance.get( + "route_unreachable_without_external_fallback_count" + ) + ), + "public_maintenance_fallback_route_count": int_value( + public_maintenance.get("maintenance_fallback_route_count") + ), "windows99_vmware_readback_present": windows99.get("readback_present") is True, "windows99_vmrun_present": windows99.get("vmrun_present") is True, "windows99_vmware_config_ready": windows99.get("config_ready") is True, @@ -1338,6 +1484,18 @@ def enrich_machine_readback(payload: dict[str, Any]) -> dict[str, Any]: "windows99_remote_execution_channel_ready" ], "windows99_ssh_batch_status": rollups["windows99_ssh_batch_status"], + "public_maintenance_runtime_readback_present": rollups[ + "public_maintenance_runtime_readback_present" + ], + "public_maintenance_fallback_ready": rollups[ + "public_maintenance_fallback_ready" + ], + "public_route_raw_5xx_without_fallback_count": rollups[ + "public_route_raw_5xx_without_fallback_count" + ], + "public_route_unreachable_without_external_fallback_count": rollups[ + "public_route_unreachable_without_external_fallback_count" + ], "runtime_write_authorized_by_this_scorecard": False, "host_reboot_authorized_by_this_scorecard": False, "workflow_trigger_authorized_by_this_scorecard": False, @@ -1371,6 +1529,15 @@ def enrich_machine_readback(payload: dict[str, Any]) -> dict[str, Any]: "reboot_auto_recovery_stockplatform_ingestion_status": rollups[ "stockplatform_ingestion_status" ], + "reboot_auto_recovery_public_maintenance_fallback_ready": rollups[ + "public_maintenance_fallback_ready" + ], + "reboot_auto_recovery_public_route_raw_5xx_without_fallback_count": rollups[ + "public_route_raw_5xx_without_fallback_count" + ], + "reboot_auto_recovery_public_route_unreachable_without_external_fallback_count": ( + rollups["public_route_unreachable_without_external_fallback_count"] + ), "reboot_auto_recovery_safe_next_step": readback["safe_next_step"], "reboot_auto_recovery_next_safe_action": readback["next_safe_action"], "reboot_auto_recovery_source_controls_present": source_controls_present, @@ -1419,6 +1586,9 @@ def build_scorecard(args: argparse.Namespace) -> dict[str, Any]: generated_at=generated_at, ) host_pressure = build_host_pressure_readback(read_json_object(args.host_pressure_file)) + public_maintenance = parse_public_maintenance_readback( + args.public_maintenance_file + ) windows99 = parse_windows99_vmware_readback(read_text(args.windows99_vmware_file)) windows99_management = parse_windows99_management_readback( args.windows99_management_file @@ -1503,6 +1673,7 @@ def build_scorecard(args: argparse.Namespace) -> dict[str, Any]: if free_gib is not None and free_gib < args.min_free_gib: blockers.append("local_disk_free_below_minimum") blockers.extend(strings(host_pressure.get("blockers"))) + blockers.extend(strings(public_maintenance.get("blockers"))) blockers.extend(strings(windows99.get("blockers"))) if ( windows99.get("readback_present") is False @@ -1521,6 +1692,7 @@ def build_scorecard(args: argparse.Namespace) -> dict[str, Any]: blockers=unique_blockers, stockplatform=stockplatform, host_pressure=host_pressure, + public_maintenance=public_maintenance, ) payload = { "schema_version": SCHEMA_VERSION, @@ -1591,6 +1763,7 @@ def build_scorecard(args: argparse.Namespace) -> dict[str, Any]: }, "stockplatform_data_freshness": stockplatform, "host_pressure": host_pressure, + "public_maintenance_fallback": public_maintenance, "windows99_vmware_autostart": windows99, "windows99_management_channel": windows99_management, "capacity": { diff --git a/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py b/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py index e1ecb19d3..107e12128 100644 --- a/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py +++ b/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py @@ -1,5 +1,8 @@ from __future__ import annotations +import json +import subprocess +import sys from pathlib import Path @@ -38,6 +41,8 @@ def test_reboot_p0_contract_has_maintenance_and_telegram_alerts() -> None: alerts = read("ops/monitoring/alerts-unified.yml") runbook = read("docs/runbooks/PUBLIC-MAINTENANCE-FALLBACK-RUNBOOK.md") snippet = read("ops/maintenance/nginx-502-maintenance-snippet.conf") + probe = read("scripts/reboot-recovery/public-maintenance-fallback-probe.py") + exporter = read("scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh") for alert_name in [ "HostRebootEventDetected", @@ -48,9 +53,131 @@ def test_reboot_p0_contract_has_maintenance_and_telegram_alerts() -> None: assert alert_name in alerts assert "notification_type: TYPE-3" in alerts assert "proxy_intercept_errors on" in snippet + assert "X-AWOOOI-Fallback" in snippet assert "502" in runbook assert "L0" in runbook assert "L1" in runbook + assert "raw_5xx_without_fallback" in probe + assert "public-maintenance-fallback.json" in exporter + assert "--public-maintenance-file" in exporter + assert "awoooi_public_route_raw_5xx_without_maintenance_fallback" in exporter + + +def test_reboot_slo_scorecard_fails_closed_on_raw_public_502(tmp_path: Path) -> None: + summary = tmp_path / "summary.txt" + summary.write_text( + "\n".join( + [ + "SERVICE_GREEN=1", + "PRODUCT_DATA_GREEN=1", + "BACKUP_CORE_GREEN=1", + "POST_START_BLOCKED=0", + "HOST_188_SERVICE_GREEN=1", + ] + ), + encoding="utf-8", + ) + host_probe = tmp_path / "host-probe.txt" + host_probe.write_text( + "\n".join( + f"HOST_BOOT alias={alias} host=192.168.0.{alias} reachable=1 uptime_seconds=120" + for alias in ["99", "110", "111", "112", "120", "121", "188"] + ), + encoding="utf-8", + ) + reboot_event = tmp_path / "reboot-event.json" + reboot_event.write_text( + json.dumps( + { + "reboot_detected": True, + "all_required_hosts_observed": True, + "all_required_hosts_in_reboot_window": True, + "fresh_boot_hosts": ["99", "110", "111", "112", "120", "121", "188"], + "missing_hosts": [], + "unreachable_hosts": [], + "target_seconds_remaining": 480, + "recovery_deadline_status": "inside_target_window", + } + ), + encoding="utf-8", + ) + public_probe = tmp_path / "public-maintenance.json" + public_probe.write_text( + json.dumps( + { + "runtime_readback_present": True, + "ready": False, + "raw_5xx_without_fallback_count": 1, + "route_unreachable_without_external_fallback_count": 0, + "maintenance_fallback_route_count": 0, + "raw_5xx_without_fallback_urls": ["https://awoooi.wooo.work/"], + "blockers": ["public_route_raw_5xx_without_maintenance_fallback"], + "routes": [ + { + "url": "https://awoooi.wooo.work/", + "http_status": 502, + "status": "raw_5xx_without_maintenance_fallback", + "fallback_header": "", + "maintenance_body_detected": False, + "maintenance_fallback_serving": False, + "raw_5xx_without_fallback": True, + } + ], + } + ), + encoding="utf-8", + ) + windows99 = tmp_path / "windows99-vmware.txt" + windows99.write_text( + "\n".join( + [ + "VMRUN_PRESENT=1", + "VMWARE_AUTOSTART_CONFIG_READY=1", + "VMWARE_AUTOSTART_POWER_READY=1", + "WINDOWS_UPDATE_NO_AUTO_REBOOT_READY=1", + "VMWARE_AUTOSTART_VERIFY_READY=1", + "VMWARE_SERVICE name=VMAuthdService present=1 status=Running startup_type=Automatic ok=1", + "VMWARE_SERVICE name=VMnetDHCP present=1 status=Running startup_type=Automatic ok=1", + "VMWARE_AUTOSTART_TASK name=AWOOOI-Start-VMware-VMs present=1 enabled=1 state=Ready trigger_count=1 action_count=1 ok=1", + "WINDOWS_UPDATE_POLICY name=NoAutoRebootWithLoggedOnUsers value=1 expected=1 ok=1", + *[ + f"VMX alias={alias} path=D:\\Documents\\Virtual Machines\\{alias}\\{alias}.vmx present=1" + for alias in ["111", "112", "120", "121", "188"] + ], + *[ + f"VM_POWER alias={alias} vmx_present=1 running=1" + for alias in ["111", "112", "120", "121", "188"] + ], + ] + ), + encoding="utf-8", + ) + + result = subprocess.run( + [ + sys.executable, + str(ROOT / "scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py"), + "--summary-file", + str(summary), + "--host-probe-file", + str(host_probe), + "--reboot-event-file", + str(reboot_event), + "--public-maintenance-file", + str(public_probe), + "--windows99-vmware-file", + str(windows99), + ], + text=True, + capture_output=True, + check=True, + ) + payload = json.loads(result.stdout) + + assert "public_route_raw_5xx_without_maintenance_fallback" in payload["active_blockers"] + assert payload["required_checks"]["public_maintenance_fallback_runtime_ready"] is False + assert payload["current_phase"] == "public_maintenance_fallback_blocked" + assert payload["rollups"]["public_route_raw_5xx_without_fallback_count"] == 1 def test_backup_exporter_emits_domain_level_backup_coverage() -> None: