fix(reboot): detect public maintenance edge drift

scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh

@@ -42,6 +42,7 @@ stock_freshness_file="$artifact_dir/stock-freshness.json"
 stock_ingestion_file="$artifact_dir/stock-ingestion.json"
 public_maintenance_file="$artifact_dir/public-maintenance-fallback.json"
 public_maintenance_prom="$artifact_dir/public-maintenance-fallback.prom"
+public_maintenance_edge_file="$artifact_dir/public-maintenance-edge-fallback.txt"
 windows99_management_file="$artifact_dir/windows99-management-channel.json"
 windows99_vmware_file="$artifact_dir/windows99-vmware-verify.txt"
 reboot_event_state_file="${REBOOT_EVENT_STATE_FILE:-${LOG_DIR}/reboot-event-state.json}"
@@ -243,6 +244,77 @@ if [ -f "$ROOT_DIR/scripts/reboot-recovery/public-maintenance-fallback-probe.py"
   python3 "${public_probe_args[@]}" >"$artifact_dir/public-maintenance-fallback.stdout" 2>"$artifact_dir/public-maintenance-fallback.err" \
     || rm -f "$public_maintenance_file" "$public_maintenance_prom"
 fi
+if [ -s "$public_maintenance_file" ] && [ -x "$ROOT_DIR/scripts/reboot-recovery/public-maintenance-edge-fallback-apply.sh" ]; then
+  bash "$ROOT_DIR/scripts/reboot-recovery/public-maintenance-edge-fallback-apply.sh" --check \
+    >"$public_maintenance_edge_file" 2>"$artifact_dir/public-maintenance-edge-fallback.err" \
+    || true
+fi
+if [ -s "$public_maintenance_file" ] && [ -s "$public_maintenance_edge_file" ]; then
+  python3 - "$public_maintenance_file" "$public_maintenance_edge_file" \
+    >"$artifact_dir/public-maintenance-edge-merge.stdout" \
+    2>"$artifact_dir/public-maintenance-edge-merge.err" <<'PY' || true
+import json
+import sys
+from pathlib import Path
+
+public_path = Path(sys.argv[1])
+edge_path = Path(sys.argv[2])
+payload = json.loads(public_path.read_text(encoding="utf-8"))
+edge_text = edge_path.read_text(encoding="utf-8", errors="replace")
+
+kv = {}
+edge_blockers = []
+for raw_line in edge_text.splitlines():
+    line = raw_line.strip()
+    if not line:
+        continue
+    if line.startswith("BLOCKER "):
+        edge_blockers.append(line.removeprefix("BLOCKER ").strip().split()[0])
+        continue
+    if "=" in line:
+        key, value = line.split("=", 1)
+        kv[key] = value
+
+edge_ready = kv.get("EDGE_FALLBACK_READY") == "1"
+edge_present = bool(kv.get("AWOOOI_PUBLIC_MAINTENANCE_EDGE_FALLBACK"))
+if edge_present and not edge_ready and not edge_blockers:
+    edge_blockers.append("edge_public_maintenance_live_config_drift")
+
+blockers = list(payload.get("blockers") or [])
+for blocker in edge_blockers:
+    if blocker and blocker not in blockers:
+        blockers.append(blocker)
+
+payload["edge_fallback_runtime_readback_present"] = edge_present
+payload["edge_fallback_ready"] = edge_ready if edge_present else None
+payload["edge_fallback_blockers"] = edge_blockers
+payload["edge_fallback"] = {
+    "runtime_readback_present": edge_present,
+    "ready": edge_ready if edge_present else None,
+    "mode": kv.get("MODE", "unknown"),
+    "host": kv.get("HOSTNAME", ""),
+    "user": kv.get("WHOAMI", ""),
+    "remote_conf": kv.get("REMOTE_CONF", ""),
+    "remote_conf_readable": kv.get("REMOTE_CONF_READABLE") == "1",
+    "live_conf_has_intercept": kv.get("LIVE_CONF_HAS_INTERCEPT") == "1",
+    "live_conf_has_error_page_503": kv.get("LIVE_CONF_HAS_ERROR_PAGE_503") == "1",
+    "live_conf_has_fallback_header": kv.get("LIVE_CONF_HAS_FALLBACK_HEADER") == "1",
+    "live_conf_has_internal_location": kv.get("LIVE_CONF_HAS_INTERNAL_LOCATION") == "1",
+    "live_conf_has_api_vip": kv.get("LIVE_CONF_HAS_API_VIP") == "1",
+    "live_conf_has_web_vip": kv.get("LIVE_CONF_HAS_WEB_VIP") == "1",
+    "maintenance_file_present": kv.get("MAINTENANCE_FILE_PRESENT") == "1",
+    "maintenance_file_has_marker": kv.get("MAINTENANCE_FILE_HAS_MARKER") == "1",
+    "sudo_n_ready": kv.get("SUDO_N_READY") == "1",
+    "blockers": edge_blockers,
+}
+payload["blockers"] = blockers
+payload["ready"] = payload.get("ready") is True and not blockers
+public_path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+print("PUBLIC_MAINTENANCE_EDGE_MERGE_DONE=1")
+print("EDGE_FALLBACK_READY=" + ("1" if edge_ready else "0"))
+print("EDGE_FALLBACK_BLOCKERS=" + ",".join(edge_blockers))
+PY
+fi
 
 if [ -f "$ROOT_DIR/scripts/reboot-recovery/windows99-management-channel-probe.py" ]; then
   python3 "$ROOT_DIR/scripts/reboot-recovery/windows99-management-channel-probe.py" \
@@ -303,6 +375,9 @@ fi
 if [ -s "$public_maintenance_file" ]; then
   scorecard_args+=(--public-maintenance-file "$public_maintenance_file")
 fi
+if [ -s "$public_maintenance_edge_file" ]; then
+  scorecard_args+=(--public-maintenance-edge-file "$public_maintenance_edge_file")
+fi
 if [ -s "$windows99_management_file" ]; then
   scorecard_args+=(--windows99-management-file "$windows99_management_file")
 fi
@@ -358,6 +433,13 @@ payload=json.load(open(sys.argv[1], encoding="utf-8"))
 print(payload.get("public_maintenance_fallback", {}).get("route_unreachable_without_external_fallback_count", 0) or 0)
 PY
 )"
+public_edge_fallback_ready="$(python3 - "$scorecard_file" <<'PY'
+import json, sys
+payload=json.load(open(sys.argv[1], encoding="utf-8"))
+value = payload.get("public_maintenance_fallback", {}).get("edge_fallback_ready")
+print(1 if value is True else 0)
+PY
+)"
 windows99_vmware_metrics="$(python3 - "$scorecard_file" <<'PY'
 import json, sys
 
@@ -514,6 +596,9 @@ awoooi_public_route_raw_5xx_without_maintenance_fallback{scope="public_routes"}
 # HELP awoooi_public_route_unreachable_without_external_l1_fallback Public routes unreachable without external fallback proof.
 # TYPE awoooi_public_route_unreachable_without_external_l1_fallback gauge
 awoooi_public_route_unreachable_without_external_l1_fallback{scope="public_routes"} $public_unreachable_count
+# HELP awoooi_public_maintenance_edge_fallback_ready Whether live edge Nginx maintenance fallback config is installed.
+# TYPE awoooi_public_maintenance_edge_fallback_ready gauge
+awoooi_public_maintenance_edge_fallback_ready{scope="public_routes"} $public_edge_fallback_ready
 # HELP awoooi_windows99_vmware_readback_present Whether Windows99 VMware verifier stdout was collected.
 # TYPE awoooi_windows99_vmware_readback_present gauge
 # HELP awoooi_windows99_vmware_verify_ready Whether Windows99 VMware autostart verifier is fully green.

scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py

@@ -49,6 +49,11 @@ def parse_args() -> argparse.Namespace:
         type=Path,
         help="Optional public maintenance fallback route probe JSON readback.",
     )
+    parser.add_argument(
+        "--public-maintenance-edge-file",
+        type=Path,
+        help="Optional public edge Nginx maintenance fallback --check output.",
+    )
     parser.add_argument(
         "--windows99-vmware-file",
         type=Path,
@@ -483,11 +488,55 @@ def read_json_object(path: Path | None) -> dict[str, Any]:
     return payload
 
 
-def parse_public_maintenance_readback(path: Path | None) -> dict[str, Any]:
+def parse_public_maintenance_edge_readback(text: str) -> dict[str, Any]:
+    """Parse no-write edge fallback check stdout."""
+    if not text.strip():
+        return {
+            "runtime_readback_present": False,
+            "ready": None,
+            "blockers": [],
+        }
+    kv = parse_kv(text)
+    blockers = []
+    for raw_line in text.splitlines():
+        line = raw_line.strip()
+        if line.startswith("BLOCKER "):
+            blockers.append(line.removeprefix("BLOCKER ").strip().split()[0])
+    ready = kv.get("EDGE_FALLBACK_READY") == "1"
+    if not ready and not blockers:
+        blockers.append("edge_public_maintenance_live_config_drift")
+    return {
+        "runtime_readback_present": True,
+        "ready": ready,
+        "mode": kv.get("MODE", "unknown"),
+        "host": kv.get("HOSTNAME", ""),
+        "user": kv.get("WHOAMI", ""),
+        "remote_conf": kv.get("REMOTE_CONF", ""),
+        "remote_conf_readable": kv.get("REMOTE_CONF_READABLE") == "1",
+        "live_conf_has_intercept": kv.get("LIVE_CONF_HAS_INTERCEPT") == "1",
+        "live_conf_has_error_page_503": kv.get("LIVE_CONF_HAS_ERROR_PAGE_503") == "1",
+        "live_conf_has_fallback_header": kv.get("LIVE_CONF_HAS_FALLBACK_HEADER") == "1",
+        "live_conf_has_internal_location": kv.get("LIVE_CONF_HAS_INTERNAL_LOCATION") == "1",
+        "live_conf_has_api_vip": kv.get("LIVE_CONF_HAS_API_VIP") == "1",
+        "live_conf_has_web_vip": kv.get("LIVE_CONF_HAS_WEB_VIP") == "1",
+        "maintenance_file_present": kv.get("MAINTENANCE_FILE_PRESENT") == "1",
+        "maintenance_file_has_marker": kv.get("MAINTENANCE_FILE_HAS_MARKER") == "1",
+        "sudo_n_ready": kv.get("SUDO_N_READY") == "1",
+        "blockers": unique_strings(blockers),
+    }
+
+
+def parse_public_maintenance_readback(
+    path: Path | None,
+    edge_path: Path | None = None,
+) -> dict[str, Any]:
     """Parse public route maintenance fallback probe JSON."""
     default = {
         "runtime_readback_present": False,
         "ready": False,
+        "edge_fallback_runtime_readback_present": False,
+        "edge_fallback_ready": None,
+        "edge_fallback_blockers": [],
         "target_url_count": 0,
         "route_count": 0,
         "raw_5xx_without_fallback_count": 0,
@@ -544,12 +593,37 @@ def parse_public_maintenance_readback(path: Path | None) -> dict[str, Any]:
         and "public_route_unreachable_without_external_l1_fallback" not in blockers
     ):
         blockers.append("public_route_unreachable_without_external_l1_fallback")
+    edge_readback = parse_public_maintenance_edge_readback(read_text(edge_path))
+    if edge_readback.get("runtime_readback_present") is not True and (
+        payload.get("edge_fallback_runtime_readback_present") is True
+    ):
+        edge_readback = {
+            "runtime_readback_present": True,
+            "ready": payload.get("edge_fallback_ready"),
+            "blockers": strings(payload.get("edge_fallback_blockers")),
+            **(
+                payload.get("edge_fallback")
+                if isinstance(payload.get("edge_fallback"), dict)
+                else {}
+            ),
+        }
+    edge_blockers = strings(edge_readback.get("blockers"))
+    for blocker in edge_blockers:
+        if blocker not in blockers:
+            blockers.append(blocker)
 
     return {
         "runtime_readback_present": True,
         "schema_version": str(payload.get("schema_version") or "unknown"),
         "generated_at": str(payload.get("generated_at") or ""),
         "ready": payload.get("ready") is True and not blockers,
+        "edge_fallback_runtime_readback_present": edge_readback.get(
+            "runtime_readback_present"
+        )
+        is True,
+        "edge_fallback_ready": edge_readback.get("ready"),
+        "edge_fallback": edge_readback,
+        "edge_fallback_blockers": edge_blockers,
         "target_url_count": int_value(payload.get("target_url_count")),
         "route_count": int_value(payload.get("route_count")) or len(compact_routes),
         "raw_5xx_without_fallback_count": raw_5xx_count,
@@ -1270,6 +1344,11 @@ def choose_safe_next_step(
             "prepare_public_maintenance_fallback_l1_external_static_origin_or_cdn_"
             "healthcheck_decision_record_then_controlled_cutover_with_rollback"
         )
+    if "edge_public_maintenance_live_config_drift" in public_blockers:
+        return (
+            "run_public_maintenance_edge_fallback_check_then_prepare_privileged_"
+            "nginx_t_apply_package_no_app_restart"
+        )
     if "public_maintenance_fallback_runtime_readback_missing" in blockers:
         return (
             "run_public_maintenance_fallback_probe_and_rerun_reboot_scorecard_"
@@ -1396,7 +1475,10 @@ def reboot_sop_current_phase(active_blockers: list[str], can_claim: bool) -> str
         "host_188_service_green_not_1",
         "wazuh_dashboard_degraded",
     }
-    if any(blocker.startswith("public_") for blocker in active_blockers):
+    if any(
+        blocker.startswith("public_") or blocker.startswith("edge_public_maintenance")
+        for blocker in active_blockers
+    ):
         return "public_maintenance_fallback_blocked"
     if any(blocker in service_blockers for blocker in active_blockers):
         return "post_reboot_service_readiness_blocked"
@@ -1439,6 +1521,7 @@ def reboot_sop_primary_blocker(active_blockers: list[str]) -> str:
         "windows99_console_verify_output_truncated",
         "public_route_raw_5xx_without_maintenance_fallback",
         "public_route_unreachable_without_external_l1_fallback",
+        "edge_public_maintenance_live_config_drift",
         "public_maintenance_fallback_runtime_readback_missing",
         "post_start_blocked_not_zero",
         "service_green_not_1",
@@ -1674,14 +1757,23 @@ def active_blocker_action_row(
         next_safe_action = "collect_wazuh_dashboard_readback_no_service_restart"
         post_verifier = "read_wazuh_dashboard_or_registry_health_readback"
         evidence_inputs = ["post_reboot_readiness.wazuh_dashboard_degraded"]
-    elif blocker.startswith("public_"):
+    elif blocker.startswith("public_") or blocker.startswith("edge_public_maintenance"):
         category = "public_maintenance_fallback"
         owner_lane = "public_route_maintenance_fallback"
-        next_safe_action = (
-            "run_public_maintenance_fallback_probe_and_rerun_scorecard_no_reload"
-        )
+        if blocker.startswith("edge_public_maintenance"):
+            next_safe_action = (
+                "run_public_maintenance_edge_fallback_check_then_prepare_privileged_"
+                "nginx_t_apply_package_no_app_restart"
+            )
+        else:
+            next_safe_action = (
+                "run_public_maintenance_fallback_probe_and_rerun_scorecard_no_reload"
+            )
         post_verifier = (
-            "python3 scripts/reboot-recovery/public-maintenance-fallback-probe.py"
+            "scripts/reboot-recovery/public-maintenance-edge-fallback-apply.sh "
+            "--check"
+            if blocker.startswith("edge_public_maintenance")
+            else "python3 scripts/reboot-recovery/public-maintenance-fallback-probe.py"
         )
         evidence_inputs = ["public_maintenance_fallback"]
         controlled_apply_mode = "check_mode_then_separate_edge_or_nginx_apply"
@@ -1907,6 +1999,12 @@ def enrich_machine_readback(payload: dict[str, Any]) -> dict[str, Any]:
         "public_route_raw_5xx_without_fallback_count": int_value(
             public_maintenance.get("raw_5xx_without_fallback_count")
         ),
+        "public_maintenance_edge_fallback_runtime_readback_present": (
+            public_maintenance.get("edge_fallback_runtime_readback_present") is True
+        ),
+        "public_maintenance_edge_fallback_ready": (
+            public_maintenance.get("edge_fallback_ready") is True
+        ),
         "public_route_unreachable_without_external_fallback_count": int_value(
             public_maintenance.get(
                 "route_unreachable_without_external_fallback_count"
@@ -2088,6 +2186,12 @@ def enrich_machine_readback(payload: dict[str, Any]) -> dict[str, Any]:
         "public_route_raw_5xx_without_fallback_count": rollups[
             "public_route_raw_5xx_without_fallback_count"
         ],
+        "public_maintenance_edge_fallback_ready": rollups[
+            "public_maintenance_edge_fallback_ready"
+        ],
+        "public_maintenance_edge_fallback_runtime_readback_present": rollups[
+            "public_maintenance_edge_fallback_runtime_readback_present"
+        ],
         "public_route_unreachable_without_external_fallback_count": rollups[
             "public_route_unreachable_without_external_fallback_count"
         ],
@@ -2130,6 +2234,9 @@ def enrich_machine_readback(payload: dict[str, Any]) -> dict[str, Any]:
         "reboot_auto_recovery_public_route_raw_5xx_without_fallback_count": rollups[
             "public_route_raw_5xx_without_fallback_count"
         ],
+        "reboot_auto_recovery_public_maintenance_edge_fallback_ready": rollups[
+            "public_maintenance_edge_fallback_ready"
+        ],
         "reboot_auto_recovery_public_route_unreachable_without_external_fallback_count": (
             rollups["public_route_unreachable_without_external_fallback_count"]
         ),
@@ -2189,7 +2296,8 @@ def build_scorecard(args: argparse.Namespace) -> dict[str, Any]:
     )
     host_pressure = build_host_pressure_readback(read_json_object(args.host_pressure_file))
     public_maintenance = parse_public_maintenance_readback(
-        args.public_maintenance_file
+        args.public_maintenance_file,
+        args.public_maintenance_edge_file,
     )
     windows99 = parse_windows99_vmware_readback(read_text(args.windows99_vmware_file))
     windows99_management = parse_windows99_management_readback(

scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_scorecard.py

@@ -129,6 +129,25 @@ PUBLIC_MAINTENANCE_GREEN = {
     ],
 }
 
+PUBLIC_MAINTENANCE_EDGE_DRIFT = """AWOOOI_PUBLIC_MAINTENANCE_EDGE_FALLBACK=1
+MODE=check
+HOSTNAME=ollama
+WHOAMI=ollama
+REMOTE_CONF=/etc/nginx/sites-enabled/awoooi.wooo.work.conf
+REMOTE_CONF_READABLE=1
+LIVE_CONF_HAS_INTERCEPT=0
+LIVE_CONF_HAS_ERROR_PAGE_503=0
+LIVE_CONF_HAS_FALLBACK_HEADER=0
+LIVE_CONF_HAS_INTERNAL_LOCATION=0
+LIVE_CONF_HAS_API_VIP=1
+LIVE_CONF_HAS_WEB_VIP=1
+MAINTENANCE_FILE_PRESENT=0
+MAINTENANCE_FILE_HAS_MARKER=0
+SUDO_N_READY=0
+EDGE_FALLBACK_READY=0
+BLOCKER edge_public_maintenance_live_config_drift
+"""
+
 
 WINDOWS99_MANAGEMENT_BLOCKED = {
     "schema_version": "windows99_management_channel_readback_v1",
@@ -282,6 +301,7 @@ def run_scorecard(
     probe: str = HOST_PROBE_GREEN,
     windows99: str = WINDOWS99_VMWARE_GREEN,
     public_maintenance: dict | None = PUBLIC_MAINTENANCE_GREEN,
+    public_maintenance_edge: str | None = None,
     windows99_management: str | None = None,
     windows99_collector: str | None = None,
 ) -> dict:
@@ -291,6 +311,7 @@ def run_scorecard(
     windows99_path = tmp_path / "windows99-vmware.txt"
     windows99_collector_path = tmp_path / "windows99-vmware-collector.txt"
     public_maintenance_path = tmp_path / "public-maintenance.json"
+    public_maintenance_edge_path = tmp_path / "public-maintenance-edge.txt"
     summary_path.write_text(summary, encoding="utf-8")
     probe_path.write_text(probe, encoding="utf-8")
     reboot_event_path.write_text(json.dumps(REBOOT_EVENT_GREEN), encoding="utf-8")
@@ -316,6 +337,14 @@ def run_scorecard(
             encoding="utf-8",
         )
         args.extend(["--public-maintenance-file", str(public_maintenance_path)])
+    if public_maintenance_edge is not None:
+        public_maintenance_edge_path.write_text(
+            public_maintenance_edge,
+            encoding="utf-8",
+        )
+        args.extend(
+            ["--public-maintenance-edge-file", str(public_maintenance_edge_path)]
+        )
     if windows99_management is not None:
         windows99_management_path.write_text(windows99_management, encoding="utf-8")
         args.extend(["--windows99-management-file", str(windows99_management_path)])
@@ -530,6 +559,40 @@ def test_missing_public_maintenance_runtime_readback_fails_closed(tmp_path: Path
     assert payload["current_phase"] == "public_maintenance_fallback_blocked"
 
 
+def test_edge_public_maintenance_live_config_drift_blocks_even_when_routes_are_200(
+    tmp_path: Path,
+) -> None:
+    payload = run_scorecard(
+        tmp_path,
+        GREEN_SUMMARY,
+        public_maintenance=PUBLIC_MAINTENANCE_GREEN,
+        public_maintenance_edge=PUBLIC_MAINTENANCE_EDGE_DRIFT,
+    )
+
+    assert payload["status"] == "blocked_reboot_auto_recovery_slo_not_ready"
+    assert "edge_public_maintenance_live_config_drift" in payload["active_blockers"]
+    assert payload["public_maintenance_fallback"]["ready"] is False
+    assert (
+        payload["public_maintenance_fallback"][
+            "edge_fallback_runtime_readback_present"
+        ]
+        is True
+    )
+    assert payload["public_maintenance_fallback"]["edge_fallback_ready"] is False
+    assert (
+        payload["public_maintenance_fallback"]["edge_fallback"]["sudo_n_ready"]
+        is False
+    )
+    assert (
+        payload["required_checks"]["public_maintenance_fallback_runtime_ready"]
+        is False
+    )
+    assert (
+        payload["rollups"]["public_maintenance_edge_fallback_ready"] is False
+    )
+    assert payload["current_phase"] == "public_maintenance_fallback_blocked"
+
+
 def test_missing_probe_fails_closed(tmp_path: Path) -> None:
     payload = run_scorecard(tmp_path, GREEN_SUMMARY, probe="")