fix(reboot): detect public maintenance edge drift

apps/api/src/services/reboot_auto_recovery_slo_scorecard.py

@@ -76,6 +76,7 @@ _PROMETHEUS_RUNTIME_COMBINED_QUERY = (
     + '"}'
 )
 _PUBLIC_MAINTENANCE_BLOCKERS = {
+    "edge_public_maintenance_live_config_drift",
     "public_maintenance_fallback_runtime_readback_missing",
     "public_route_raw_5xx_without_maintenance_fallback",
     "public_route_unreachable_without_external_l1_fallback",
@@ -1318,6 +1319,12 @@ def apply_public_maintenance_runtime_readback(
     readback["public_route_raw_5xx_without_fallback_count"] = _int(
         public_maintenance.get("raw_5xx_without_fallback_count")
     )
+    readback["public_maintenance_edge_fallback_ready"] = (
+        public_maintenance.get("edge_fallback_ready") is True
+    )
+    readback["public_maintenance_edge_fallback_runtime_readback_present"] = (
+        public_maintenance.get("edge_fallback_runtime_readback_present") is True
+    )
 
     rollups = _dict(payload.setdefault("rollups", {}))
     rollups["active_blocker_count"] = active_blocker_count
@@ -1332,6 +1339,12 @@ def apply_public_maintenance_runtime_readback(
     rollups["public_route_raw_5xx_without_fallback_count"] = _int(
         public_maintenance.get("raw_5xx_without_fallback_count")
     )
+    rollups["public_maintenance_edge_fallback_ready"] = (
+        public_maintenance.get("edge_fallback_ready") is True
+    )
+    rollups["public_maintenance_edge_fallback_runtime_readback_present"] = (
+        public_maintenance.get("edge_fallback_runtime_readback_present") is True
+    )
     rollups["public_route_unreachable_without_external_fallback_count"] = _int(
         public_maintenance.get("route_unreachable_without_external_fallback_count")
     )
@@ -1826,6 +1839,12 @@ def _build_payload(scorecard: dict[str, Any], path: Path) -> dict[str, Any]:
         "public_route_raw_5xx_without_fallback_count": _int(
             public_maintenance.get("raw_5xx_without_fallback_count")
         ),
+        "public_maintenance_edge_fallback_ready": (
+            public_maintenance.get("edge_fallback_ready") is True
+        ),
+        "public_maintenance_edge_fallback_runtime_readback_present": (
+            public_maintenance.get("edge_fallback_runtime_readback_present") is True
+        ),
         "public_route_unreachable_without_external_fallback_count": _int(
             public_maintenance.get(
                 "route_unreachable_without_external_fallback_count"
@@ -2092,6 +2111,9 @@ def _build_payload(scorecard: dict[str, Any], path: Path) -> dict[str, Any]:
             "public_route_raw_5xx_without_fallback_count": rollups[
                 "public_route_raw_5xx_without_fallback_count"
             ],
+            "public_maintenance_edge_fallback_ready": rollups[
+                "public_maintenance_edge_fallback_ready"
+            ],
             "controlled_service_data_backup_readback_status": rollups[
                 "controlled_service_data_backup_readback_status"
             ],
@@ -2232,6 +2254,9 @@ def _build_public_maintenance_fallback_readback(value: Any) -> dict[str, Any]:
         return {
             "runtime_readback_present": False,
             "ready": False,
+            "edge_fallback_runtime_readback_present": False,
+            "edge_fallback_ready": None,
+            "edge_fallback_blockers": [],
             "target_url_count": 0,
             "route_count": 0,
             "raw_5xx_without_fallback_count": 0,
@@ -2259,12 +2284,22 @@ def _build_public_maintenance_fallback_readback(value: Any) -> dict[str, Any]:
         and "public_route_unreachable_without_external_l1_fallback" not in blockers
     ):
         blockers.append("public_route_unreachable_without_external_l1_fallback")
+    edge_blockers = _strings(public_maintenance.get("edge_fallback_blockers"))
+    for blocker in edge_blockers:
+        if blocker not in blockers:
+            blockers.append(blocker)
 
     return {
         "runtime_readback_present": True,
         "schema_version": str(public_maintenance.get("schema_version") or "unknown"),
         "generated_at": str(public_maintenance.get("generated_at") or ""),
         "ready": public_maintenance.get("ready") is True and not blockers,
+        "edge_fallback_runtime_readback_present": (
+            public_maintenance.get("edge_fallback_runtime_readback_present") is True
+        ),
+        "edge_fallback_ready": public_maintenance.get("edge_fallback_ready"),
+        "edge_fallback": _dict(public_maintenance.get("edge_fallback")),
+        "edge_fallback_blockers": edge_blockers,
         "target_url_count": _int(public_maintenance.get("target_url_count")),
         "route_count": _int(public_maintenance.get("route_count")),
         "raw_5xx_without_fallback_count": raw_5xx_count,
@@ -3317,6 +3352,7 @@ def _reboot_sop_primary_blocker(active_blockers: list[str]) -> str:
         "windows99_console_verify_output_truncated",
         "public_route_raw_5xx_without_maintenance_fallback",
         "public_route_unreachable_without_external_l1_fallback",
+        "edge_public_maintenance_live_config_drift",
         "public_maintenance_fallback_runtime_readback_missing",
         "post_start_blocked_not_zero",
         "service_green_not_1",

apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py

@@ -80,6 +80,34 @@ PUBLIC_MAINTENANCE_RUNTIME_READY = {
     ],
 }
 
+PUBLIC_MAINTENANCE_RUNTIME_EDGE_DRIFT = {
+    **PUBLIC_MAINTENANCE_RUNTIME_READY,
+    "ready": False,
+    "edge_fallback_runtime_readback_present": True,
+    "edge_fallback_ready": False,
+    "edge_fallback_blockers": ["edge_public_maintenance_live_config_drift"],
+    "edge_fallback": {
+        "runtime_readback_present": True,
+        "ready": False,
+        "mode": "check",
+        "host": "ollama",
+        "user": "ollama",
+        "remote_conf": "/etc/nginx/sites-enabled/awoooi.wooo.work.conf",
+        "remote_conf_readable": True,
+        "live_conf_has_intercept": False,
+        "live_conf_has_error_page_503": False,
+        "live_conf_has_fallback_header": False,
+        "live_conf_has_internal_location": False,
+        "live_conf_has_api_vip": True,
+        "live_conf_has_web_vip": True,
+        "maintenance_file_present": False,
+        "maintenance_file_has_marker": False,
+        "sudo_n_ready": False,
+        "blockers": ["edge_public_maintenance_live_config_drift"],
+    },
+    "blockers": ["edge_public_maintenance_live_config_drift"],
+}
+
 _REPO_ROOT = Path(__file__).resolve().parents[3]
 _SOURCE_SCORECARD = (
     _REPO_ROOT
@@ -201,6 +229,44 @@ def test_reboot_auto_recovery_slo_scorecard_does_not_reopen_ready_public_mainten
     ] == 1
 
 
+def test_reboot_auto_recovery_slo_scorecard_overlays_edge_fallback_live_drift(
+    tmp_path,
+):
+    runtime_path = tmp_path / "public-maintenance-fallback.json"
+    runtime_path.write_text(
+        json.dumps(PUBLIC_MAINTENANCE_RUNTIME_EDGE_DRIFT),
+        encoding="utf-8",
+    )
+
+    payload = load_latest_reboot_auto_recovery_slo_scorecard(
+        public_maintenance_runtime_path=runtime_path
+    )
+
+    assert "edge_public_maintenance_live_config_drift" in payload["active_blockers"]
+    assert payload["public_maintenance_fallback"]["ready"] is False
+    assert (
+        payload["public_maintenance_fallback"][
+            "edge_fallback_runtime_readback_present"
+        ]
+        is True
+    )
+    assert payload["public_maintenance_fallback"]["edge_fallback_ready"] is False
+    assert (
+        payload["public_maintenance_fallback"]["edge_fallback"]["sudo_n_ready"]
+        is False
+    )
+    assert payload["readback"]["public_maintenance_edge_fallback_ready"] is False
+    assert (
+        payload["readback"][
+            "public_maintenance_edge_fallback_runtime_readback_present"
+        ]
+        is True
+    )
+    assert payload["required_checks"][
+        "public_maintenance_fallback_runtime_ready"
+    ] is False
+
+
 def test_reboot_auto_recovery_slo_scorecard_reconciles_green_runtime_readback_metric_blockers():
     metric_readback = dict(PROMETHEUS_RUNTIME_READBACK)
     metric_readback["active_blockers"] = [