kdrive/modesetting: Work around mesa format restrictions for dumb gbm buffers

If a dumb gbm buffer cannot be created for a given depth,
but a tiled buffer works, use the tiled buffer and hope it works

Signed-off-by: stefan11111 <stefan11111github@gmail.com>

hw/kdrive/modesetting/modesetting.c

@@ -15,7 +15,7 @@
 #endif
 
 struct gbm_bo*
-modesetting_open(KdScreenInfo *screen, Bool need_map, Bool keep_depth)
+modesetting_open(KdScreenInfo *screen, Bool need_map, Bool keep_depth, Bool probe)
 {
     struct gbm_bo *ret = NULL;
 
@@ -26,11 +26,20 @@ modesetting_open(KdScreenInfo *screen, Bool need_map, Bool keep_depth)
 
     if (screen->dumb) {
         need_map = TRUE;
+        probe = FALSE;
     } else if (config->no_tile) {
         need_map = TRUE;
+        probe = FALSE;
     } else if (randr != RR_Rotate_0) {
         /* TODO: Can we do better? */
         need_map = TRUE;
+        probe = FALSE;
+    } else {
+        msPriv *priv = screen->card->driver;
+        const char *name = gbm_device_get_backend_name(priv->gbm);
+        if (!name || strcmp(name, "drm")) {
+            probe = FALSE;
+        }
     }
 #endif
 
@@ -44,6 +53,18 @@ modesetting_open(KdScreenInfo *screen, Bool need_map, Bool keep_depth)
             ret = gbm_create_front_for_screen(screen, TRUE /* do_map */, config->format_swap);
         }
 
+        /*
+         * Mesa's gbm backend does not currently allow creating dumb buffers
+         * with format other than GBM_FORMAT_{X,A}RGB8888
+         *
+         * See if a tiled front buffer can be created, and hope that it will work with glamor.
+         */
+#ifdef GLAMOR
+        if (!ret && probe && need_map) {
+            ret = gbm_create_front_for_screen(screen, FALSE /* do_map */, config->format_swap);
+        }
+#endif
+
         if (!ret) {
             int old_depth = screen->fb.depth;
             int old_bpp = screen->fb.bitsPerPixel;
@@ -237,7 +258,7 @@ msScreenInitialize(KdScreenInfo * screen, msScrPriv * scrpriv)
         screen->height = scrpriv->mode ? scrpriv->mode->vdisplay : 1080;
     }
 
-    scrpriv->front = modesetting_open(screen, TRUE /* need_map */, FALSE /* keep_depth */);
+    scrpriv->front = modesetting_open(screen, TRUE /* need_map */, FALSE /* keep_depth */, TRUE /* probe */);
     if (!scrpriv->front) {
         LogMessage(X_ERROR, "Xmodesetting(card %d, screen %d): Could not create a front buffer\n",
                    screen->card->mynum, screen->mynum);

hw/kdrive/modesetting/modesetting.h

@@ -120,7 +120,7 @@ Bool msUnmapFramebuffer(KdScreenInfo * screen);
 Bool msSetShadow(ScreenPtr pScreen);
 
 struct gbm_bo*
-modesetting_open(KdScreenInfo *screen, Bool need_map, Bool keep_depth);
+modesetting_open(KdScreenInfo *screen, Bool need_map, Bool keep_depth, Bool probe);
 
 /* ms_cursor.c */
 

hw/kdrive/modesetting/ms_glamor.c

@@ -17,35 +17,44 @@ msGlamorTileFront(ScreenPtr pScreen)
 {
     KdScreenPriv(pScreen);
     KdScreenInfo *screen = pScreenPriv->screen;
+    msScrPriv *scrpriv = screen->driver;
 
     KdFrameBuffer saved_framebuffer = screen->fb;
 
     Bool is_gles = glamor_is_gles(pScreen);
+    Bool flip = FALSE;
 
     struct gbm_bo *new_front = NULL;
 
     /* By design, kdrive depth and bpp can't change between initAccel and finiAccel
      * https://www.x.org/Development/Documentation/KdriveDrivers/
      */
-    new_front = modesetting_open(screen, FALSE /* need_map */, TRUE /* keep_depth */);
+    new_front = modesetting_open(screen, FALSE /* need_map */, TRUE /* keep_depth */, FALSE /* probe */);
     if (!new_front) {
-        return FALSE;
+        if (!gbm_bo_get_map(scrpriv->front)) {
+            /* Try to texture the old front */
+            flip = TRUE;
+        } else {
+            return FALSE;
+        }
     }
 
     /* Maybe we could switch to this bo, but why? */
-    if (gbm_bo_get_map(new_front)) {
+    if (new_front && gbm_bo_get_map(new_front)) {
         gbm_bo_destroy(new_front);
         return FALSE;
     }
 
-    if (!msSetScreenBo(pScreen, new_front, FALSE /* flip */)) {
-        gbm_bo_destroy(new_front);
+    if (!msSetScreenBo(pScreen, new_front ? new_front : scrpriv->front, flip)) {
+        if (new_front) {
+            gbm_bo_destroy(new_front);
+        }
         LogMessage(X_ERROR, "Xmodesetting(%d): Could not swap to the new front bo\n", pScreen->myNum);
         return FALSE;
     }
 
     /* Update visual masks if needed */
-    gbm_bo_set_screen_fb_info(new_front, screen, is_gles);
+    gbm_bo_set_screen_fb_info(new_front ? new_front : scrpriv->front, screen, is_gles);
     if (memcmp(&saved_framebuffer, &screen->fb, sizeof(screen->fb))) {
         /* TODO: Make this generic
          * For now, the only thing that can happen is r-b masks need swapping
@@ -82,6 +91,11 @@ msGlamorCreateRes(ScreenPtr pScreen)
     if (!screen->dumb) {
         if (!msGlamorTileFront(pScreen)) {
             LogMessage(X_INFO, "Xmodesetting(%d): Could not create a usable tiled front buffer\n", pScreen->myNum);
+            if (!gbm_bo_get_map(scrpriv->front)) {
+                LogMessage(X_ERROR, "Xmodesetting(%d): Cannot run at this depth/bpp\n", pScreen->myNum);
+                LogMessage(X_ERROR, "Xmodesetting(%d): Choose a different depth/bpp or use the dumb gbm backend\n", pScreen->myNum);
+                return FALSE;
+            }
         }
     }