From ff9e086e2e58f7b0d4aff473449a6f7fd681a77d Mon Sep 17 00:00:00 2001 From: Your Name Date: Fri, 3 Jul 2026 14:50:11 +0800 Subject: [PATCH] fix(reboot): detect public maintenance edge drift --- .../reboot_auto_recovery_slo_scorecard.py | 36 +++++ ..._reboot_auto_recovery_slo_scorecard_api.py | 66 ++++++++++ docs/LOGBOOK.md | 25 ++++ docs/runbooks/FULL-STACK-COLD-START-SOP.md | 4 +- docs/runbooks/REBOOT-RECOVERY-SOP.md | 12 +- ...oot-cold-start-backup-recovery-workplan.md | 2 +- ops/monitoring/alerts-unified.yml | 16 +++ .../reboot-auto-recovery-slo-exporter.sh | 85 ++++++++++++ .../reboot-auto-recovery-slo-scorecard.py | 124 ++++++++++++++++-- ...test_reboot_auto_recovery_slo_scorecard.py | 63 +++++++++ 10 files changed, 421 insertions(+), 12 deletions(-) diff --git a/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py b/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py index e4937699a..12386a756 100644 --- a/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py +++ b/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py @@ -76,6 +76,7 @@ _PROMETHEUS_RUNTIME_COMBINED_QUERY = ( + '"}' ) _PUBLIC_MAINTENANCE_BLOCKERS = { + "edge_public_maintenance_live_config_drift", "public_maintenance_fallback_runtime_readback_missing", "public_route_raw_5xx_without_maintenance_fallback", "public_route_unreachable_without_external_l1_fallback", @@ -1318,6 +1319,12 @@ def apply_public_maintenance_runtime_readback( readback["public_route_raw_5xx_without_fallback_count"] = _int( public_maintenance.get("raw_5xx_without_fallback_count") ) + readback["public_maintenance_edge_fallback_ready"] = ( + public_maintenance.get("edge_fallback_ready") is True + ) + readback["public_maintenance_edge_fallback_runtime_readback_present"] = ( + public_maintenance.get("edge_fallback_runtime_readback_present") is True + ) rollups = _dict(payload.setdefault("rollups", {})) rollups["active_blocker_count"] = active_blocker_count @@ -1332,6 +1339,12 @@ def apply_public_maintenance_runtime_readback( rollups["public_route_raw_5xx_without_fallback_count"] = _int( public_maintenance.get("raw_5xx_without_fallback_count") ) + rollups["public_maintenance_edge_fallback_ready"] = ( + public_maintenance.get("edge_fallback_ready") is True + ) + rollups["public_maintenance_edge_fallback_runtime_readback_present"] = ( + public_maintenance.get("edge_fallback_runtime_readback_present") is True + ) rollups["public_route_unreachable_without_external_fallback_count"] = _int( public_maintenance.get("route_unreachable_without_external_fallback_count") ) @@ -1826,6 +1839,12 @@ def _build_payload(scorecard: dict[str, Any], path: Path) -> dict[str, Any]: "public_route_raw_5xx_without_fallback_count": _int( public_maintenance.get("raw_5xx_without_fallback_count") ), + "public_maintenance_edge_fallback_ready": ( + public_maintenance.get("edge_fallback_ready") is True + ), + "public_maintenance_edge_fallback_runtime_readback_present": ( + public_maintenance.get("edge_fallback_runtime_readback_present") is True + ), "public_route_unreachable_without_external_fallback_count": _int( public_maintenance.get( "route_unreachable_without_external_fallback_count" @@ -2092,6 +2111,9 @@ def _build_payload(scorecard: dict[str, Any], path: Path) -> dict[str, Any]: "public_route_raw_5xx_without_fallback_count": rollups[ "public_route_raw_5xx_without_fallback_count" ], + "public_maintenance_edge_fallback_ready": rollups[ + "public_maintenance_edge_fallback_ready" + ], "controlled_service_data_backup_readback_status": rollups[ "controlled_service_data_backup_readback_status" ], @@ -2232,6 +2254,9 @@ def _build_public_maintenance_fallback_readback(value: Any) -> dict[str, Any]: return { "runtime_readback_present": False, "ready": False, + "edge_fallback_runtime_readback_present": False, + "edge_fallback_ready": None, + "edge_fallback_blockers": [], "target_url_count": 0, "route_count": 0, "raw_5xx_without_fallback_count": 0, @@ -2259,12 +2284,22 @@ def _build_public_maintenance_fallback_readback(value: Any) -> dict[str, Any]: and "public_route_unreachable_without_external_l1_fallback" not in blockers ): blockers.append("public_route_unreachable_without_external_l1_fallback") + edge_blockers = _strings(public_maintenance.get("edge_fallback_blockers")) + for blocker in edge_blockers: + if blocker not in blockers: + blockers.append(blocker) return { "runtime_readback_present": True, "schema_version": str(public_maintenance.get("schema_version") or "unknown"), "generated_at": str(public_maintenance.get("generated_at") or ""), "ready": public_maintenance.get("ready") is True and not blockers, + "edge_fallback_runtime_readback_present": ( + public_maintenance.get("edge_fallback_runtime_readback_present") is True + ), + "edge_fallback_ready": public_maintenance.get("edge_fallback_ready"), + "edge_fallback": _dict(public_maintenance.get("edge_fallback")), + "edge_fallback_blockers": edge_blockers, "target_url_count": _int(public_maintenance.get("target_url_count")), "route_count": _int(public_maintenance.get("route_count")), "raw_5xx_without_fallback_count": raw_5xx_count, @@ -3317,6 +3352,7 @@ def _reboot_sop_primary_blocker(active_blockers: list[str]) -> str: "windows99_console_verify_output_truncated", "public_route_raw_5xx_without_maintenance_fallback", "public_route_unreachable_without_external_l1_fallback", + "edge_public_maintenance_live_config_drift", "public_maintenance_fallback_runtime_readback_missing", "post_start_blocked_not_zero", "service_green_not_1", diff --git a/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py b/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py index 035f1989f..4d8ea78e1 100644 --- a/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py +++ b/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py @@ -80,6 +80,34 @@ PUBLIC_MAINTENANCE_RUNTIME_READY = { ], } +PUBLIC_MAINTENANCE_RUNTIME_EDGE_DRIFT = { + **PUBLIC_MAINTENANCE_RUNTIME_READY, + "ready": False, + "edge_fallback_runtime_readback_present": True, + "edge_fallback_ready": False, + "edge_fallback_blockers": ["edge_public_maintenance_live_config_drift"], + "edge_fallback": { + "runtime_readback_present": True, + "ready": False, + "mode": "check", + "host": "ollama", + "user": "ollama", + "remote_conf": "/etc/nginx/sites-enabled/awoooi.wooo.work.conf", + "remote_conf_readable": True, + "live_conf_has_intercept": False, + "live_conf_has_error_page_503": False, + "live_conf_has_fallback_header": False, + "live_conf_has_internal_location": False, + "live_conf_has_api_vip": True, + "live_conf_has_web_vip": True, + "maintenance_file_present": False, + "maintenance_file_has_marker": False, + "sudo_n_ready": False, + "blockers": ["edge_public_maintenance_live_config_drift"], + }, + "blockers": ["edge_public_maintenance_live_config_drift"], +} + _REPO_ROOT = Path(__file__).resolve().parents[3] _SOURCE_SCORECARD = ( _REPO_ROOT @@ -201,6 +229,44 @@ def test_reboot_auto_recovery_slo_scorecard_does_not_reopen_ready_public_mainten ] == 1 +def test_reboot_auto_recovery_slo_scorecard_overlays_edge_fallback_live_drift( + tmp_path, +): + runtime_path = tmp_path / "public-maintenance-fallback.json" + runtime_path.write_text( + json.dumps(PUBLIC_MAINTENANCE_RUNTIME_EDGE_DRIFT), + encoding="utf-8", + ) + + payload = load_latest_reboot_auto_recovery_slo_scorecard( + public_maintenance_runtime_path=runtime_path + ) + + assert "edge_public_maintenance_live_config_drift" in payload["active_blockers"] + assert payload["public_maintenance_fallback"]["ready"] is False + assert ( + payload["public_maintenance_fallback"][ + "edge_fallback_runtime_readback_present" + ] + is True + ) + assert payload["public_maintenance_fallback"]["edge_fallback_ready"] is False + assert ( + payload["public_maintenance_fallback"]["edge_fallback"]["sudo_n_ready"] + is False + ) + assert payload["readback"]["public_maintenance_edge_fallback_ready"] is False + assert ( + payload["readback"][ + "public_maintenance_edge_fallback_runtime_readback_present" + ] + is True + ) + assert payload["required_checks"][ + "public_maintenance_fallback_runtime_ready" + ] is False + + def test_reboot_auto_recovery_slo_scorecard_reconciles_green_runtime_readback_metric_blockers(): metric_readback = dict(PROMETHEUS_RUNTIME_READBACK) metric_readback["active_blockers"] = [ diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index ab2f6771a..70f728d65 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -19,6 +19,31 @@ **下一步**： - commit / push 到 Gitea main；等待 CD deploy marker 後讀回 production `/api/v1/agents/reboot-auto-recovery-slo-scorecard`、`/api/v1/agents/windows99-vmware-autostart-check-mode-package` 與 `/api/v1/agents/awoooi-priority-work-order-readback`，確認 autostart package 欄位在 production 可見。 +## 2026-07-03 — 14:40 Public maintenance edge fallback drift 主動偵測 + +**完成內容**： +- 最新 main `3ebeeeb4b fix(knowledge): reduce knowledge base text density` 已由 Gitea CD `#4592` 推 deploy marker `ee0da93 chore(cd): deploy 3ebeeeb [skip ci]`；deploy readback matched、job succeeded，但 run-level UI 一度仍顯示 Running lag。 +- CD / deploy window 期間 production `https://awoooi.wooo.work/api/v1/health` 與 `/api/v1/agents/reboot-auto-recovery-slo-scorecard` 讀到 raw `502 Bad Gateway`，無 `X-AWOOOI-Fallback`、無 `Retry-After`、無 maintenance body；稍後 API 自癒回 200，SLO scorecard 仍為 7 個真 blocker。 +- `public-maintenance-edge-fallback-apply.sh --check` 讀回 live 188 edge drift：`LIVE_CONF_HAS_INTERCEPT=0`、`LIVE_CONF_HAS_ERROR_PAGE_503=0`、`LIVE_CONF_HAS_FALLBACK_HEADER=0`、`LIVE_CONF_HAS_INTERNAL_LOCATION=0`、`MAINTENANCE_FILE_PRESENT=0`、`SUDO_N_READY=0`、`EDGE_FALLBACK_READY=0`、`BLOCKER edge_public_maintenance_live_config_drift`。 +- `reboot-auto-recovery-slo-exporter.sh` 現在會執行 edge fallback `--check`，把結果 merge 進 `public-maintenance-fallback.json`，並輸出 `awoooi_public_maintenance_edge_fallback_ready{scope="public_routes"}` metric。 +- CLI / API scorecard 已支援 `edge_public_maintenance_live_config_drift`：即使 public routes 當下都是 200，只要 live edge fallback 沒套用，`public_maintenance_fallback.ready=false`、`required_checks.public_maintenance_fallback_runtime_ready=false`，並進 active blocker / action matrix。 +- `ops/monitoring/alerts-unified.yml` 新增 `PublicMaintenanceEdgeFallbackConfigDrift`，讓 edge fallback drift 進 Telegram / Alertmanager 告警鏈，不再等使用者看到 raw 502。 +- `FULL-STACK-COLD-START-SOP` 升到 v1.122，`REBOOT-RECOVERY-SOP` 升到 v5.14，workplan P0-2 更新為 `SOURCE_READY_RUNTIME_EDGE_DRIFT_BLOCKED`。 + +**已跑驗證**： +- `DATABASE_URL=postgresql+asyncpg://test:test@localhost/test PYTHONPATH=apps/api python3.11 -m pytest scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_scorecard.py apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py -q -p no:cacheprovider`：`37 passed`。 +- `python3.11 -m py_compile scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py apps/api/src/services/reboot_auto_recovery_slo_scorecard.py`：通過。 +- `bash -n scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh scripts/reboot-recovery/public-maintenance-edge-fallback-apply.sh`：通過。 +- `python3.11 ops/runner/guard-gitea-runner-pressure.py`：`GITEA_RUNNER_PRESSURE_GUARD_OK workflow_files=12 scheduled_workflows=4 auto_branch_events_on_110=0 generic_runner_labels=0`。 +- `git diff --check`：通過。 + +**仍維持**： +- 這一輪只把 P0-2 edge fallback drift 接成主動偵測 / metric / alert / scorecard blocker；未執行 `--apply`、未 reload Nginx、未重啟 app / Docker / host、未讀密碼或 secret。 +- P0-2 runtime 完成條件仍是具備 privileged edge apply channel 後執行受控 `--apply`、備份 live conf、`nginx -t`、reload、public route header verifier 全綠，或完成外部 L1 cloud/CDN fallback。 + +**下一步**： +- 部署此 source，重跑 110 SLO installer，使 110 textfile 產生 `awoooi_public_maintenance_edge_fallback_ready=0` 與 API active blocker；再回到 P0-006 host reboot / Windows99 guest power blocker。 + ## 2026-07-03 — 14:26 Windows99 runtime metric overlay production readback 收斂 **完成內容**： diff --git a/docs/runbooks/FULL-STACK-COLD-START-SOP.md b/docs/runbooks/FULL-STACK-COLD-START-SOP.md index a07f1ec75..2acac3342 100644 --- a/docs/runbooks/FULL-STACK-COLD-START-SOP.md +++ b/docs/runbooks/FULL-STACK-COLD-START-SOP.md @@ -1,6 +1,6 @@ # AWOOOI 全棧冷啟動與主機重啟 SOP -> Version: v1.121 +> Version: v1.122 > Last updated: 2026-07-03 Asia/Taipei > Scope: 99 / 110 / 111 / 112 / 120 / 121 / 188 全棧重啟恢復。112 仍是 Kali / VM guest 訊號，但 2026-06-30 全主機重啟後已納入 10 分鐘 SLO 的必要 boot / power signal；此納入不代表授權任何破壞性 runtime apply。 @@ -74,6 +74,8 @@ v1.120 Windows99 runtime metric overlay precedence rule：v1.110 的 nested bloc v1.121 deployed Windows99 runtime metric overlay readback rule：2026-07-03 14:26 production readback 已證明 source `33c391808` 的 API overlay 與 110 textfile metric 對齊。Gitea CD `#4589` 為 `Success`，deploy marker `9f49651`，production deploy readback matched；110 最新 SLO run `20260703-142122` 回 `Result=success`，textfile metric 為 `awoooi_reboot_auto_recovery_slo_blocker_count=7`、`awoooi_windows99_vmware_autostart_config_ready=1`、`awoooi_windows99_vmware_windows_update_no_auto_reboot_ready=1`、missing VMX alias 空、powered-off aliases `110 / 112 / 120 / 121 / 188`。production `/api/v1/agents/reboot-auto-recovery-slo-scorecard` 必須讀回 `active_blocker_count=7`、`runtime_metric_active_blocker_count=7`、`runtime_metric_runtime_readback_added_blocker_count=0`、`readiness_percent=67`、`primary_blocker=reboot_event_required_host_unreachable`；若再次出現 Windows99 config / Windows Update policy blocker，視為 API / Prometheus / textfile regression 先修 readback，不得跳去 power on VM、改 Windows policy、重啟 host 或重啟服務。這不代表 10 分鐘自動恢復 SLO 完成；剩餘真 blocker 仍是 fresh all-host reboot event / 10-minute window / host reachability / Windows99 guest power readback。 +v1.122 public maintenance edge fallback drift metric rule：deploy / reboot window 期間 AWOOOI API / SLO route 只要曾回 raw `502` 且缺 `X-AWOOOI-Fallback` / `Retry-After` / maintenance body，即使稍後自癒為 `200`，仍要保留 P0 blocker。`reboot-auto-recovery-slo-exporter.sh` 必須執行 `public-maintenance-edge-fallback-apply.sh --check`，把 live 188 edge Nginx drift merge 進 `public-maintenance-fallback.json`，並輸出 `awoooi_public_maintenance_edge_fallback_ready{scope="public_routes"}`。當 edge check 顯示 `EDGE_FALLBACK_READY=0` 時，scorecard / API 必須新增 `edge_public_maintenance_live_config_drift`，讓 `public_maintenance_fallback.ready=false`；Alertmanager 必須透過 `PublicMaintenanceEdgeFallbackConfigDrift` 告警。此規則只授權 no-write check / metric / alert；實際 `--apply` 必須有 passwordless sudo、backup、`nginx -t`、reload 與 public header verifier，不得讀密碼、不得重啟 app / Docker / host。 + 2026-07-02 110 control-path / Harbor recovery receipt rule：若 Gitea Harbor repair queue 仍保留 `harbor_110_remote_ssh_publickey_auth_stalled`、remote-control unavailable、jobs stale 或 historical failure，但同一輪本地證據同時證明 `wooo` command path ready、110 local Harbor `/v2/` ready、public/internal registry `/v2/` 回 `401`，則該 Gitea Harbor repair 失敗只能列為 historical queue metadata，不得再當成 current SSH blocker。必須用 `/api/v1/agents/harbor-registry-controlled-recovery-receipt` 或同等 validator 合併 `diagnose-110-ssh-publickey-auth.sh`、`recover-110-control-path-and-harbor-local.sh --check`、public Gitea queue readback 與 registry `/v2/` verifier，並把機器可讀結果寫入 `docs/operations/harbor-110-control-path-recovery-readback-2026-07-02.snapshot.json` 類型的 snapshot。2026-07-02 live receipt 顯示：public/internal registry `/v2/` 均為 `401`、latest visible CD `#4335` 為 `Success`、Gitea Harbor repair failure 已是 `historical_after_latest_cd_success=true`；active blockers 收斂為 110 controlled CD lane config / binary / registration / service guardrail、active action container pressure，以及 Gitea CD jobs head-SHA / stale readback mismatch。若 local-console output 只有 `AWOOOI_110_CONTROLLED_CD_LANE_READY` marker，non110 runner parser 不得從 110 `BLOCKER` 行推導 non110 blocker；non110 只有看到 `AWOOOI_NON110_RUNNER_READY` marker 才能列入 active blocker。 2026-07-02 110 controlled CD lane fail-closed enforcer staging rule：110 runner 壓力事故後，legacy / generic runner 仍必須 fail-closed；但 `awoooi-cd-lane-drain.service` 的非 secret staging artifact 不得再被 enforcer 無差別封回 stub。`scripts/reboot-recovery/enforce-110-runner-failclosed.sh` 只有在 `config.yaml` 符合 `capacity <= 1`、只含 `awoooi-host:host` 與 `awoooi-ubuntu:docker://192.168.0.110:5000/awoooi/ci-runner:act-22.04`、binary 是 executable ELF、systemd unit 具備 `ConditionPathExists=/home/wooo/awoooi-cd-lane-drain/data/.runner`、`CPUAccounting` / `MemoryAccounting` / `TasksAccounting` / `NoNewPrivileges` 等 guardrail，且 service `inactive`、`MainPID=0`、未 enabled / 未 masked 時，才可保留 drain config / binary / unit，並輸出 `CONTROLLED_DRAIN_STAGING_ALLOWED=1` 與 textfile metric。此 staging 規則不得讀 token、不得讀 `.runner` 內容、不得註冊 runner、不得啟動 service；若 registration 缺失，readiness verifier 仍必須只留下 `controlled_cd_lane_registration_missing` / `controlled_cd_lane_service_not_active` 類 blocker。若 `CONTROLLED_DRAIN_STAGING_ALLOWED=0` 且 config / binary 又被搬走，優先修 source enforcer / unit guardrail，不要手工反覆補同一組 artifact。 diff --git a/docs/runbooks/REBOOT-RECOVERY-SOP.md b/docs/runbooks/REBOOT-RECOVERY-SOP.md index 6208f8c0d..801d0ccf6 100644 --- a/docs/runbooks/REBOOT-RECOVERY-SOP.md +++ b/docs/runbooks/REBOOT-RECOVERY-SOP.md @@ -1,9 +1,9 @@ # AWOOOI 重開機恢復 SOP -> **版本**: v5.13 +> **版本**: v5.14 > **最後更新**: 2026-07-03 (台北時間) > **更新者**: Codex -> **觸發事件**: 2026-07-03 14:26 Windows99 runtime metric overlay 已部署並 production readback 對齊：API / runtime metric 均為 7 個真 blocker，Windows99 config / Windows Update 假 blocker 已清除 +> **觸發事件**: 2026-07-03 14:40 public maintenance edge fallback drift 已接入 SLO exporter / scorecard / alert；raw 502 不再等人工發現 --- @@ -21,6 +21,14 @@ ## 架構概覽與依賴圖 +### 2026-07-03 14:40 P0-2 Public Maintenance 現況覆蓋 + +最新 main `3ebeeeb4b` 的 Gitea CD `#4592` 已推 deploy marker `ee0da93`，production deploy readback matched、job succeeded；deploy window 期間 AWOOOI API health 與 SLO route 曾回 raw `502 Bad Gateway`，沒有 `X-AWOOOI-Fallback`、`Retry-After` 或維護頁 body。後續 public route probe 自癒回 `200`，但 raw 502 exposure 仍必須列為 P0 public UX blocker。 + +live 188 edge check 的目前事實是：`LIVE_CONF_HAS_INTERCEPT=0`、`LIVE_CONF_HAS_ERROR_PAGE_503=0`、`LIVE_CONF_HAS_FALLBACK_HEADER=0`、`LIVE_CONF_HAS_INTERNAL_LOCATION=0`、`MAINTENANCE_FILE_PRESENT=0`、`SUDO_N_READY=0`、`EDGE_FALLBACK_READY=0`，blocker 固定為 `edge_public_maintenance_live_config_drift`。此 blocker 不授權 app restart、Docker restart、host reboot 或讀密碼；只能先 `public-maintenance-edge-fallback-apply.sh --check`，具備 passwordless sudo 後才可進受控 `--apply` + backup + `nginx -t` + reload + public header verifier，或走 L1 external static/CDN fallback decision record。 + +v5.14 起，110 SLO exporter 必須把 edge check 合併到 public maintenance readback，輸出 `awoooi_public_maintenance_edge_fallback_ready{scope="public_routes"}`；production scorecard / API 必須在 routes 當下 200 但 edge fallback 未套用時，仍顯示 `public_maintenance_fallback.ready=false` 與 active blocker `edge_public_maintenance_live_config_drift`。Alertmanager 規則 `PublicMaintenanceEdgeFallbackConfigDrift` 必須對此 metric 告警，避免下一次 deploy / reboot raw 502 才由使用者發現。 + ### 2026-07-03 14:26 P0 現況覆蓋 Gitea CD `#4589` 已完成 source `33c391808` 部署，deploy marker `9f49651 chore(cd): deploy 33c3918 [skip ci]` 已 fast-forward 回 `gitea-ssh/main`；public queue readback 顯示 `current_main_cd_run_status=Success`、`current_main_cd_deploy_evidence_succeeded_from_log=true`、`latest_visible_cd_production_deploy_readback_matched_from_log=true`、`latest_visible_cd_job_succeeded_from_log=true`、`no_matching_online_runner_visible=false`，且沒有 host pressure refusal。110 `awoooi-reboot-auto-recovery-slo.service` 最新 run `20260703-142122` 已 `Result=success`、`ExecMainStatus=0`，textfile metric 顯示 `awoooi_reboot_auto_recovery_slo_blocker_count=7`、`awoooi_windows99_vmware_autostart_config_ready=1`、`awoooi_windows99_vmware_windows_update_no_auto_reboot_ready=1`、`awoooi_windows99_vmware_missing_vmx_alias` 無 alias gauge，powered-off aliases 為 `110 / 112 / 120 / 121 / 188`。 diff --git a/docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md b/docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md index 22790ffa5..52eeb3155 100644 --- a/docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md +++ b/docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md @@ -20,7 +20,7 @@ | 優先 | 狀態 | 工作項 | 最新證據 | 下一步 / 完成條件 | |------|------|--------|----------|-------------------| | P0-1 | BLOCKED_HOST_WINDOWS | 全主機 reboot auto-detection / auto-trigger / 10 分鐘恢復 SLO | 2026-07-03 10:44 production scorecard：`status=blocked_reboot_auto_recovery_slo_not_ready`、`active_blocker_count=10`、`readiness_percent=60`、active blockers 仍含 reboot event / host unreachable / 99+111+Windows99 VMX / guest power，且新增 `windows99_vmware_autostart_config_not_ready`、`windows99_update_no_auto_reboot_policy_not_ready`。no-write artifact `/tmp/awoooi-reboot-continue-20260703-085205`：111 socket probe 仍 timeout / unreachable；99 RDP / VMConnect reachable 但 uptime unknown；188 socket probe reachable，前次 host probe 仍為 `systemd_state=degraded` / `startup_active=failed`。 | 先收斂 99 / Windows99 / VMware 與 111：恢復 no-secret management channel 或取得 validator 可接受的完整 console Verify stdout，讀回 VMX / VM power / host uptime，再 rerun host probe + reboot-event detector；不得 reboot、不得 VM power change、不得讀 Windows 密碼，不得用 RDP clipboard 片段當完成證據。 | -| P0-2 | BLOCKED_EDGE_PRIVILEGED_APPLY | Deploy / reboot 期間 public 502 維護頁與外部 fallback | Gitea CD `#4519` 已推 deploy marker `3aca484 -> a94ddd5`，但 marker 後 public probe 仍讀到 `https://awoooi.wooo.work/api/v1/health` raw `502`、fallback header/body 空；live 188 `/etc/nginx/sites-enabled/awoooi.wooo.work.conf` 缺 maintenance fallback，`/var/www/maintenance/maintenance.html` 缺失，`ollama@188` 無 passwordless sudo。 | 先跑 `scripts/reboot-recovery/public-maintenance-edge-fallback-apply.sh --check` 留 drift receipt；具備 privileged channel 後執行 `--apply`，要求 backup、`nginx -t`、reload、public route probe 全綠。不得讀密碼、不得用 app restart 掩蓋 edge fallback drift。 | +| P0-2 | SOURCE_READY_RUNTIME_EDGE_DRIFT_BLOCKED | Deploy / reboot 期間 public 502 維護頁與外部 fallback | 2026-07-03 最新 main `3ebeeeb4b` 的 Gitea CD `#4592` 已推 deploy marker `ee0da93` 且 deploy readback matched，但 deploy window 期間 AWOOOI API health / SLO route 仍出現 raw `502`、無 fallback header/body。live 188 edge check 固定 drift：`LIVE_CONF_HAS_INTERCEPT=0`、`LIVE_CONF_HAS_ERROR_PAGE_503=0`、`LIVE_CONF_HAS_FALLBACK_HEADER=0`、`LIVE_CONF_HAS_INTERNAL_LOCATION=0`、`MAINTENANCE_FILE_PRESENT=0`、`SUDO_N_READY=0`、`EDGE_FALLBACK_READY=0`。source 已新增 `edge_public_maintenance_live_config_drift` scorecard blocker、`awoooi_public_maintenance_edge_fallback_ready` metric 與 `PublicMaintenanceEdgeFallbackConfigDrift` alert rule。 | 部署新版 exporter / API 後，110 textfile 與 production scorecard 必須主動顯示 edge fallback drift blocker；具備 privileged channel 後才可執行受控 `public-maintenance-edge-fallback-apply.sh --apply`，要求 backup、`nginx -t`、reload、public header verifier 全綠；或完成 L1 external cloud/CDN fallback。不得讀密碼、不得用 app / Docker / host restart 掩蓋 edge fallback drift。 | | P0-3 | PARTIAL_GREEN_DEPLOYED_SOURCE | 所有產品 / 網站版本與資料最新性 | Gitea `main=bc8d3af3f chore(cd): deploy dacdd90 [skip ci]`；CD `#4558` log 顯示 API/Web build/push wrapper 全部 `rc=0`、deploy marker `dacdd90`、production deploy readback matched。Production SLO route 已讀回最新 API 行為：`runtime_metric_runtime_readback_added_blockers=["windows99_vmware_autostart_config_not_ready","windows99_update_no_auto_reboot_policy_not_ready"]`、`windows99_update_no_auto_reboot_ready=false`。Stock freshness 仍為 `status=ok`、`latest_trading_date=2026-07-02`、blockers `[]`。 | 版本層已補到 AWOOOI production；完成整體 P0 仍需收斂 Windows99 VMX / service / policy / power 與 host reboot-event blockers。後續每個 public product 仍要維持 source SHA、deploy marker、runtime endpoint、public route watch、freshness 四層一致。 | | P0-4 | BLOCKED_WINDOWS99_AUTOSTART | 192.168.0.99 VMware 自動啟動與 VM guest 111 / 188 / 120 / 121 / 112 | 10:44 production readback 已把 nested Windows99 evidence 上卷到 active blockers：`windows99_vmware_vmx_missing`、`windows99_vmware_guest_power_not_ready`、`windows99_vmware_autostart_config_not_ready`、`windows99_update_no_auto_reboot_policy_not_ready`；`runtime_metric_runtime_readback_added_blockers` 含 config / policy 兩項，`windows99_update_no_auto_reboot_ready=false`。09:53 no-secret management probe：99 reachable、RDP / Hyper-V VMConnect reachable，WinRM timeout，Mac vantage collector 仍為 `ssh_batchmode_auth_ready=0` / `blocked_ssh_publickey_auth_missing`；completion 仍取決於 VMX / service / policy / power 全綠。 | 先修 111 VMX source 與 VMware service / policy readback，使 check-mode package 能解釋 powered-off aliases；不得從 scorecard 直接重啟 Windows service、改 registry、啟動 / 關閉 VM 或 host reboot。完成條件是 VMX config ready、guest power ready、99 uptime known、all required host reachable，且 active blocker matrix 顯示 service / policy / VMX / power 分開收斂。 | | P0-5 | RUNTIME_READY_BACKUP_RECEIPT_GAP | Gitea / 主機 / DB / 網站 / 服務 / 套件 / 工具 / log 備份監控告警 | Gitea repo bundle readback ready：expected `12`、rows `12`、missing `0`、failed `0`、sample restore dry-run ok；backup core green。2026-07-03 source / runtime 已部署 `awoooi_backup_alert_receipt_*` 指標與 Prometheus rules；110 exporter 讀回 88 個 stage requirement、188 讀回 12 個 stage requirement，`BackupAlertReceiptMetricMissing*` inactive，`BackupAlertReceiptStageMissing` 已修成每 `host / receipt_channel` 聚合 pending：110 一條、188 一條。 | 補 `/backup/alert-receipts/*.last_success` 脫敏 marker；下一層仍要補 Gitea full dump、DB/settings/issues/packages/LFS、所有工具與 log 全量備份監控。 | diff --git a/ops/monitoring/alerts-unified.yml b/ops/monitoring/alerts-unified.yml index 6bf6f133e..a788d864f 100644 --- a/ops/monitoring/alerts-unified.yml +++ b/ops/monitoring/alerts-unified.yml @@ -1169,6 +1169,22 @@ groups: description: "重啟或 upstream failure 期間 public route 仍露出 5xx；應由 L0 Nginx maintenance fallback 或 L1 external cloud/CDN 維護頁接手。" runbook: "依 docs/runbooks/PUBLIC-MAINTENANCE-FALLBACK-RUNBOOK.md 部署 ops/maintenance/maintenance.html 與 Nginx 502/503/504 intercept；整段 edge unreachable 時啟用外部 CDN/static fallback。" + - alert: PublicMaintenanceEdgeFallbackConfigDrift + expr: awoooi_public_maintenance_edge_fallback_ready{scope="public_routes"} == 0 + for: 3m + labels: + severity: critical + layer: external + component: public-maintenance-fallback + team: ops + alert_category: reboot_recovery + notification_type: TYPE-3 + auto_repair: "false" + annotations: + summary: "Public maintenance L0 edge fallback 尚未套用" + description: "SLO exporter 已讀到 live edge Nginx / maintenance file 尚未具備 502/503/504 fallback；部署或重啟期間會露出 raw 5xx。" + runbook: "先讀最新 /home/wooo/reboot-recovery/reboot-auto-recovery-slo-*/public-maintenance-edge-fallback.txt；只允許 public-maintenance-edge-fallback-apply.sh --check 或具備 passwordless sudo 的受控 --apply + nginx -t + header verifier，不得重啟 app / Docker / host。" + - alert: ExternalSiteSSLExpiringSoon expr: probe_ssl_earliest_cert_expiry{job="blackbox-http"} - time() < 14 * 24 * 3600 for: 1h diff --git a/scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh b/scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh index be1be5072..197461ee6 100755 --- a/scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh +++ b/scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh @@ -42,6 +42,7 @@ stock_freshness_file="$artifact_dir/stock-freshness.json" stock_ingestion_file="$artifact_dir/stock-ingestion.json" public_maintenance_file="$artifact_dir/public-maintenance-fallback.json" public_maintenance_prom="$artifact_dir/public-maintenance-fallback.prom" +public_maintenance_edge_file="$artifact_dir/public-maintenance-edge-fallback.txt" windows99_management_file="$artifact_dir/windows99-management-channel.json" windows99_vmware_file="$artifact_dir/windows99-vmware-verify.txt" reboot_event_state_file="${REBOOT_EVENT_STATE_FILE:-${LOG_DIR}/reboot-event-state.json}" @@ -243,6 +244,77 @@ if [ -f "$ROOT_DIR/scripts/reboot-recovery/public-maintenance-fallback-probe.py" python3 "${public_probe_args[@]}" >"$artifact_dir/public-maintenance-fallback.stdout" 2>"$artifact_dir/public-maintenance-fallback.err" \ || rm -f "$public_maintenance_file" "$public_maintenance_prom" fi +if [ -s "$public_maintenance_file" ] && [ -x "$ROOT_DIR/scripts/reboot-recovery/public-maintenance-edge-fallback-apply.sh" ]; then + bash "$ROOT_DIR/scripts/reboot-recovery/public-maintenance-edge-fallback-apply.sh" --check \ + >"$public_maintenance_edge_file" 2>"$artifact_dir/public-maintenance-edge-fallback.err" \ + || true +fi +if [ -s "$public_maintenance_file" ] && [ -s "$public_maintenance_edge_file" ]; then + python3 - "$public_maintenance_file" "$public_maintenance_edge_file" \ + >"$artifact_dir/public-maintenance-edge-merge.stdout" \ + 2>"$artifact_dir/public-maintenance-edge-merge.err" <<'PY' || true +import json +import sys +from pathlib import Path + +public_path = Path(sys.argv[1]) +edge_path = Path(sys.argv[2]) +payload = json.loads(public_path.read_text(encoding="utf-8")) +edge_text = edge_path.read_text(encoding="utf-8", errors="replace") + +kv = {} +edge_blockers = [] +for raw_line in edge_text.splitlines(): + line = raw_line.strip() + if not line: + continue + if line.startswith("BLOCKER "): + edge_blockers.append(line.removeprefix("BLOCKER ").strip().split()[0]) + continue + if "=" in line: + key, value = line.split("=", 1) + kv[key] = value + +edge_ready = kv.get("EDGE_FALLBACK_READY") == "1" +edge_present = bool(kv.get("AWOOOI_PUBLIC_MAINTENANCE_EDGE_FALLBACK")) +if edge_present and not edge_ready and not edge_blockers: + edge_blockers.append("edge_public_maintenance_live_config_drift") + +blockers = list(payload.get("blockers") or []) +for blocker in edge_blockers: + if blocker and blocker not in blockers: + blockers.append(blocker) + +payload["edge_fallback_runtime_readback_present"] = edge_present +payload["edge_fallback_ready"] = edge_ready if edge_present else None +payload["edge_fallback_blockers"] = edge_blockers +payload["edge_fallback"] = { + "runtime_readback_present": edge_present, + "ready": edge_ready if edge_present else None, + "mode": kv.get("MODE", "unknown"), + "host": kv.get("HOSTNAME", ""), + "user": kv.get("WHOAMI", ""), + "remote_conf": kv.get("REMOTE_CONF", ""), + "remote_conf_readable": kv.get("REMOTE_CONF_READABLE") == "1", + "live_conf_has_intercept": kv.get("LIVE_CONF_HAS_INTERCEPT") == "1", + "live_conf_has_error_page_503": kv.get("LIVE_CONF_HAS_ERROR_PAGE_503") == "1", + "live_conf_has_fallback_header": kv.get("LIVE_CONF_HAS_FALLBACK_HEADER") == "1", + "live_conf_has_internal_location": kv.get("LIVE_CONF_HAS_INTERNAL_LOCATION") == "1", + "live_conf_has_api_vip": kv.get("LIVE_CONF_HAS_API_VIP") == "1", + "live_conf_has_web_vip": kv.get("LIVE_CONF_HAS_WEB_VIP") == "1", + "maintenance_file_present": kv.get("MAINTENANCE_FILE_PRESENT") == "1", + "maintenance_file_has_marker": kv.get("MAINTENANCE_FILE_HAS_MARKER") == "1", + "sudo_n_ready": kv.get("SUDO_N_READY") == "1", + "blockers": edge_blockers, +} +payload["blockers"] = blockers +payload["ready"] = payload.get("ready") is True and not blockers +public_path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") +print("PUBLIC_MAINTENANCE_EDGE_MERGE_DONE=1") +print("EDGE_FALLBACK_READY=" + ("1" if edge_ready else "0")) +print("EDGE_FALLBACK_BLOCKERS=" + ",".join(edge_blockers)) +PY +fi if [ -f "$ROOT_DIR/scripts/reboot-recovery/windows99-management-channel-probe.py" ]; then python3 "$ROOT_DIR/scripts/reboot-recovery/windows99-management-channel-probe.py" \ @@ -303,6 +375,9 @@ fi if [ -s "$public_maintenance_file" ]; then scorecard_args+=(--public-maintenance-file "$public_maintenance_file") fi +if [ -s "$public_maintenance_edge_file" ]; then + scorecard_args+=(--public-maintenance-edge-file "$public_maintenance_edge_file") +fi if [ -s "$windows99_management_file" ]; then scorecard_args+=(--windows99-management-file "$windows99_management_file") fi @@ -358,6 +433,13 @@ payload=json.load(open(sys.argv[1], encoding="utf-8")) print(payload.get("public_maintenance_fallback", {}).get("route_unreachable_without_external_fallback_count", 0) or 0) PY )" +public_edge_fallback_ready="$(python3 - "$scorecard_file" <<'PY' +import json, sys +payload=json.load(open(sys.argv[1], encoding="utf-8")) +value = payload.get("public_maintenance_fallback", {}).get("edge_fallback_ready") +print(1 if value is True else 0) +PY +)" windows99_vmware_metrics="$(python3 - "$scorecard_file" <<'PY' import json, sys @@ -514,6 +596,9 @@ awoooi_public_route_raw_5xx_without_maintenance_fallback{scope="public_routes"} # HELP awoooi_public_route_unreachable_without_external_l1_fallback Public routes unreachable without external fallback proof. # TYPE awoooi_public_route_unreachable_without_external_l1_fallback gauge awoooi_public_route_unreachable_without_external_l1_fallback{scope="public_routes"} $public_unreachable_count +# HELP awoooi_public_maintenance_edge_fallback_ready Whether live edge Nginx maintenance fallback config is installed. +# TYPE awoooi_public_maintenance_edge_fallback_ready gauge +awoooi_public_maintenance_edge_fallback_ready{scope="public_routes"} $public_edge_fallback_ready # HELP awoooi_windows99_vmware_readback_present Whether Windows99 VMware verifier stdout was collected. # TYPE awoooi_windows99_vmware_readback_present gauge # HELP awoooi_windows99_vmware_verify_ready Whether Windows99 VMware autostart verifier is fully green. diff --git a/scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py b/scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py index 6a22d10b8..50c445351 100755 --- a/scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py +++ b/scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py @@ -49,6 +49,11 @@ def parse_args() -> argparse.Namespace: type=Path, help="Optional public maintenance fallback route probe JSON readback.", ) + parser.add_argument( + "--public-maintenance-edge-file", + type=Path, + help="Optional public edge Nginx maintenance fallback --check output.", + ) parser.add_argument( "--windows99-vmware-file", type=Path, @@ -483,11 +488,55 @@ def read_json_object(path: Path | None) -> dict[str, Any]: return payload -def parse_public_maintenance_readback(path: Path | None) -> dict[str, Any]: +def parse_public_maintenance_edge_readback(text: str) -> dict[str, Any]: + """Parse no-write edge fallback check stdout.""" + if not text.strip(): + return { + "runtime_readback_present": False, + "ready": None, + "blockers": [], + } + kv = parse_kv(text) + blockers = [] + for raw_line in text.splitlines(): + line = raw_line.strip() + if line.startswith("BLOCKER "): + blockers.append(line.removeprefix("BLOCKER ").strip().split()[0]) + ready = kv.get("EDGE_FALLBACK_READY") == "1" + if not ready and not blockers: + blockers.append("edge_public_maintenance_live_config_drift") + return { + "runtime_readback_present": True, + "ready": ready, + "mode": kv.get("MODE", "unknown"), + "host": kv.get("HOSTNAME", ""), + "user": kv.get("WHOAMI", ""), + "remote_conf": kv.get("REMOTE_CONF", ""), + "remote_conf_readable": kv.get("REMOTE_CONF_READABLE") == "1", + "live_conf_has_intercept": kv.get("LIVE_CONF_HAS_INTERCEPT") == "1", + "live_conf_has_error_page_503": kv.get("LIVE_CONF_HAS_ERROR_PAGE_503") == "1", + "live_conf_has_fallback_header": kv.get("LIVE_CONF_HAS_FALLBACK_HEADER") == "1", + "live_conf_has_internal_location": kv.get("LIVE_CONF_HAS_INTERNAL_LOCATION") == "1", + "live_conf_has_api_vip": kv.get("LIVE_CONF_HAS_API_VIP") == "1", + "live_conf_has_web_vip": kv.get("LIVE_CONF_HAS_WEB_VIP") == "1", + "maintenance_file_present": kv.get("MAINTENANCE_FILE_PRESENT") == "1", + "maintenance_file_has_marker": kv.get("MAINTENANCE_FILE_HAS_MARKER") == "1", + "sudo_n_ready": kv.get("SUDO_N_READY") == "1", + "blockers": unique_strings(blockers), + } + + +def parse_public_maintenance_readback( + path: Path | None, + edge_path: Path | None = None, +) -> dict[str, Any]: """Parse public route maintenance fallback probe JSON.""" default = { "runtime_readback_present": False, "ready": False, + "edge_fallback_runtime_readback_present": False, + "edge_fallback_ready": None, + "edge_fallback_blockers": [], "target_url_count": 0, "route_count": 0, "raw_5xx_without_fallback_count": 0, @@ -544,12 +593,37 @@ def parse_public_maintenance_readback(path: Path | None) -> dict[str, Any]: and "public_route_unreachable_without_external_l1_fallback" not in blockers ): blockers.append("public_route_unreachable_without_external_l1_fallback") + edge_readback = parse_public_maintenance_edge_readback(read_text(edge_path)) + if edge_readback.get("runtime_readback_present") is not True and ( + payload.get("edge_fallback_runtime_readback_present") is True + ): + edge_readback = { + "runtime_readback_present": True, + "ready": payload.get("edge_fallback_ready"), + "blockers": strings(payload.get("edge_fallback_blockers")), + **( + payload.get("edge_fallback") + if isinstance(payload.get("edge_fallback"), dict) + else {} + ), + } + edge_blockers = strings(edge_readback.get("blockers")) + for blocker in edge_blockers: + if blocker not in blockers: + blockers.append(blocker) return { "runtime_readback_present": True, "schema_version": str(payload.get("schema_version") or "unknown"), "generated_at": str(payload.get("generated_at") or ""), "ready": payload.get("ready") is True and not blockers, + "edge_fallback_runtime_readback_present": edge_readback.get( + "runtime_readback_present" + ) + is True, + "edge_fallback_ready": edge_readback.get("ready"), + "edge_fallback": edge_readback, + "edge_fallback_blockers": edge_blockers, "target_url_count": int_value(payload.get("target_url_count")), "route_count": int_value(payload.get("route_count")) or len(compact_routes), "raw_5xx_without_fallback_count": raw_5xx_count, @@ -1270,6 +1344,11 @@ def choose_safe_next_step( "prepare_public_maintenance_fallback_l1_external_static_origin_or_cdn_" "healthcheck_decision_record_then_controlled_cutover_with_rollback" ) + if "edge_public_maintenance_live_config_drift" in public_blockers: + return ( + "run_public_maintenance_edge_fallback_check_then_prepare_privileged_" + "nginx_t_apply_package_no_app_restart" + ) if "public_maintenance_fallback_runtime_readback_missing" in blockers: return ( "run_public_maintenance_fallback_probe_and_rerun_reboot_scorecard_" @@ -1396,7 +1475,10 @@ def reboot_sop_current_phase(active_blockers: list[str], can_claim: bool) -> str "host_188_service_green_not_1", "wazuh_dashboard_degraded", } - if any(blocker.startswith("public_") for blocker in active_blockers): + if any( + blocker.startswith("public_") or blocker.startswith("edge_public_maintenance") + for blocker in active_blockers + ): return "public_maintenance_fallback_blocked" if any(blocker in service_blockers for blocker in active_blockers): return "post_reboot_service_readiness_blocked" @@ -1439,6 +1521,7 @@ def reboot_sop_primary_blocker(active_blockers: list[str]) -> str: "windows99_console_verify_output_truncated", "public_route_raw_5xx_without_maintenance_fallback", "public_route_unreachable_without_external_l1_fallback", + "edge_public_maintenance_live_config_drift", "public_maintenance_fallback_runtime_readback_missing", "post_start_blocked_not_zero", "service_green_not_1", @@ -1674,14 +1757,23 @@ def active_blocker_action_row( next_safe_action = "collect_wazuh_dashboard_readback_no_service_restart" post_verifier = "read_wazuh_dashboard_or_registry_health_readback" evidence_inputs = ["post_reboot_readiness.wazuh_dashboard_degraded"] - elif blocker.startswith("public_"): + elif blocker.startswith("public_") or blocker.startswith("edge_public_maintenance"): category = "public_maintenance_fallback" owner_lane = "public_route_maintenance_fallback" - next_safe_action = ( - "run_public_maintenance_fallback_probe_and_rerun_scorecard_no_reload" - ) + if blocker.startswith("edge_public_maintenance"): + next_safe_action = ( + "run_public_maintenance_edge_fallback_check_then_prepare_privileged_" + "nginx_t_apply_package_no_app_restart" + ) + else: + next_safe_action = ( + "run_public_maintenance_fallback_probe_and_rerun_scorecard_no_reload" + ) post_verifier = ( - "python3 scripts/reboot-recovery/public-maintenance-fallback-probe.py" + "scripts/reboot-recovery/public-maintenance-edge-fallback-apply.sh " + "--check" + if blocker.startswith("edge_public_maintenance") + else "python3 scripts/reboot-recovery/public-maintenance-fallback-probe.py" ) evidence_inputs = ["public_maintenance_fallback"] controlled_apply_mode = "check_mode_then_separate_edge_or_nginx_apply" @@ -1907,6 +1999,12 @@ def enrich_machine_readback(payload: dict[str, Any]) -> dict[str, Any]: "public_route_raw_5xx_without_fallback_count": int_value( public_maintenance.get("raw_5xx_without_fallback_count") ), + "public_maintenance_edge_fallback_runtime_readback_present": ( + public_maintenance.get("edge_fallback_runtime_readback_present") is True + ), + "public_maintenance_edge_fallback_ready": ( + public_maintenance.get("edge_fallback_ready") is True + ), "public_route_unreachable_without_external_fallback_count": int_value( public_maintenance.get( "route_unreachable_without_external_fallback_count" @@ -2088,6 +2186,12 @@ def enrich_machine_readback(payload: dict[str, Any]) -> dict[str, Any]: "public_route_raw_5xx_without_fallback_count": rollups[ "public_route_raw_5xx_without_fallback_count" ], + "public_maintenance_edge_fallback_ready": rollups[ + "public_maintenance_edge_fallback_ready" + ], + "public_maintenance_edge_fallback_runtime_readback_present": rollups[ + "public_maintenance_edge_fallback_runtime_readback_present" + ], "public_route_unreachable_without_external_fallback_count": rollups[ "public_route_unreachable_without_external_fallback_count" ], @@ -2130,6 +2234,9 @@ def enrich_machine_readback(payload: dict[str, Any]) -> dict[str, Any]: "reboot_auto_recovery_public_route_raw_5xx_without_fallback_count": rollups[ "public_route_raw_5xx_without_fallback_count" ], + "reboot_auto_recovery_public_maintenance_edge_fallback_ready": rollups[ + "public_maintenance_edge_fallback_ready" + ], "reboot_auto_recovery_public_route_unreachable_without_external_fallback_count": ( rollups["public_route_unreachable_without_external_fallback_count"] ), @@ -2189,7 +2296,8 @@ def build_scorecard(args: argparse.Namespace) -> dict[str, Any]: ) host_pressure = build_host_pressure_readback(read_json_object(args.host_pressure_file)) public_maintenance = parse_public_maintenance_readback( - args.public_maintenance_file + args.public_maintenance_file, + args.public_maintenance_edge_file, ) windows99 = parse_windows99_vmware_readback(read_text(args.windows99_vmware_file)) windows99_management = parse_windows99_management_readback( diff --git a/scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_scorecard.py b/scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_scorecard.py index bf02cfa1a..cd61b1f54 100644 --- a/scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_scorecard.py +++ b/scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_scorecard.py @@ -129,6 +129,25 @@ PUBLIC_MAINTENANCE_GREEN = { ], } +PUBLIC_MAINTENANCE_EDGE_DRIFT = """AWOOOI_PUBLIC_MAINTENANCE_EDGE_FALLBACK=1 +MODE=check +HOSTNAME=ollama +WHOAMI=ollama +REMOTE_CONF=/etc/nginx/sites-enabled/awoooi.wooo.work.conf +REMOTE_CONF_READABLE=1 +LIVE_CONF_HAS_INTERCEPT=0 +LIVE_CONF_HAS_ERROR_PAGE_503=0 +LIVE_CONF_HAS_FALLBACK_HEADER=0 +LIVE_CONF_HAS_INTERNAL_LOCATION=0 +LIVE_CONF_HAS_API_VIP=1 +LIVE_CONF_HAS_WEB_VIP=1 +MAINTENANCE_FILE_PRESENT=0 +MAINTENANCE_FILE_HAS_MARKER=0 +SUDO_N_READY=0 +EDGE_FALLBACK_READY=0 +BLOCKER edge_public_maintenance_live_config_drift +""" + WINDOWS99_MANAGEMENT_BLOCKED = { "schema_version": "windows99_management_channel_readback_v1", @@ -282,6 +301,7 @@ def run_scorecard( probe: str = HOST_PROBE_GREEN, windows99: str = WINDOWS99_VMWARE_GREEN, public_maintenance: dict | None = PUBLIC_MAINTENANCE_GREEN, + public_maintenance_edge: str | None = None, windows99_management: str | None = None, windows99_collector: str | None = None, ) -> dict: @@ -291,6 +311,7 @@ def run_scorecard( windows99_path = tmp_path / "windows99-vmware.txt" windows99_collector_path = tmp_path / "windows99-vmware-collector.txt" public_maintenance_path = tmp_path / "public-maintenance.json" + public_maintenance_edge_path = tmp_path / "public-maintenance-edge.txt" summary_path.write_text(summary, encoding="utf-8") probe_path.write_text(probe, encoding="utf-8") reboot_event_path.write_text(json.dumps(REBOOT_EVENT_GREEN), encoding="utf-8") @@ -316,6 +337,14 @@ def run_scorecard( encoding="utf-8", ) args.extend(["--public-maintenance-file", str(public_maintenance_path)]) + if public_maintenance_edge is not None: + public_maintenance_edge_path.write_text( + public_maintenance_edge, + encoding="utf-8", + ) + args.extend( + ["--public-maintenance-edge-file", str(public_maintenance_edge_path)] + ) if windows99_management is not None: windows99_management_path.write_text(windows99_management, encoding="utf-8") args.extend(["--windows99-management-file", str(windows99_management_path)]) @@ -530,6 +559,40 @@ def test_missing_public_maintenance_runtime_readback_fails_closed(tmp_path: Path assert payload["current_phase"] == "public_maintenance_fallback_blocked" +def test_edge_public_maintenance_live_config_drift_blocks_even_when_routes_are_200( + tmp_path: Path, +) -> None: + payload = run_scorecard( + tmp_path, + GREEN_SUMMARY, + public_maintenance=PUBLIC_MAINTENANCE_GREEN, + public_maintenance_edge=PUBLIC_MAINTENANCE_EDGE_DRIFT, + ) + + assert payload["status"] == "blocked_reboot_auto_recovery_slo_not_ready" + assert "edge_public_maintenance_live_config_drift" in payload["active_blockers"] + assert payload["public_maintenance_fallback"]["ready"] is False + assert ( + payload["public_maintenance_fallback"][ + "edge_fallback_runtime_readback_present" + ] + is True + ) + assert payload["public_maintenance_fallback"]["edge_fallback_ready"] is False + assert ( + payload["public_maintenance_fallback"]["edge_fallback"]["sudo_n_ready"] + is False + ) + assert ( + payload["required_checks"]["public_maintenance_fallback_runtime_ready"] + is False + ) + assert ( + payload["rollups"]["public_maintenance_edge_fallback_ready"] is False + ) + assert payload["current_phase"] == "public_maintenance_fallback_blocked" + + def test_missing_probe_fails_closed(tmp_path: Path) -> None: payload = run_scorecard(tmp_path, GREEN_SUMMARY, probe="")