kdrive/modesetting: Fix screen rotations when the buffer is tiled

When the front buffer is tiled, this untextures the root pixmap,
and replaces the tiled bo with a cpu-mapped bo (+ ShadowFB if needed)

In the future, we should probably create a tiled shadow buffer,
and use gl for rotations

Signed-off-by: stefan11111 <stefan11111github@gmail.com>

hw/kdrive/modesetting/modesetting.c

@@ -29,6 +29,7 @@ modesetting_open(KdScreenInfo *screen, Bool need_map, Bool keep_depth)
     } else if (config->no_tile) {
         need_map = TRUE;
     } else if (randr != RR_Rotate_0) {
+        /* TODO: Can we do better? */
         need_map = TRUE;
     }
 #endif

hw/kdrive/modesetting/ms_randr.c

@@ -42,28 +42,45 @@ msSetMode(ScreenPtr pScreen, int width, int height, int rate)
     screen->height = height;
     screen->rate = rate;
 
-    /* Create a new front with the new sizes */
-    if (width != old_width ||
-        height != old_height) {
+    /*
+     * Create a new front with the new sizes
+     * XXX At least for now we need a new bo even for same-size rotations
+     */
+    {
         uint32_t format = gbm_bo_get_format(scrpriv->front);
-        Bool do_map = !!gbm_bo_get_map(scrpriv->front);
+        Bool do_map = (scrpriv->randr != RR_Rotate_0) || !!gbm_bo_get_map(scrpriv->front);
         uint64_t modifier = gbm_bo_get_modifier(scrpriv->front);
+        Bool used_modifiers = gbm_bo_get_used_modifiers(scrpriv->front);
 
         /* Try the current modifier first */
-        new_front = gbm_create_front_bo(priv->gbm, do_map, width, height, format,
-                                        &modifier, 1);
+        if (used_modifiers && !new_front) {
+            new_front = gbm_create_front_bo(priv->gbm, do_map, width, height, format,
+                                            &modifier, 1);
+        }
+
         if (!new_front) {
             new_front = gbm_create_front_bo(priv->gbm, do_map, width, height, format,
                                             modifiers, num_modifiers);
         }
+
+        if (!used_modifiers && !new_front) {
+            new_front = gbm_create_front_bo(priv->gbm, do_map, width, height, format,
+                                            &modifier, 1);
+        }
+
+        /*
+         * If we need to map, we don't need the video card to be able to render to this bo
+         * XXX What should we do if we don't?
+         */
+        if (!new_front) {
+            new_front = gbm_create_front_bo(priv->gbm, do_map, width, height, format,
+                                            NULL, 0);
+        }
+
         if (!new_front ||
             !msSetScreenBo(pScreen, new_front, FALSE /* flip */)) {
             goto bail;
         }
-    } else {
-        if (!msSetScreenBo(pScreen, scrpriv->front, TRUE /* flip */)) {
-            goto bail;
-        }
     }
 
     return TRUE;

hw/kdrive/modesetting/ms_util.c

@@ -9,6 +9,7 @@
 
 #ifdef GLAMOR
 #include "glamor.h"
+#include "glamor_egl.h" /* for glamor_egl_untexture_pixmap */
 #endif
 
 /* XXX This really belongs in os/, like the version from glamor_egl */
@@ -98,6 +99,8 @@ msSetScreenBo(ScreenPtr pScreen, struct gbm_bo *bo, Bool flip)
     KdScreenInfo *screen = pScreenPriv->screen;
     msScrPriv *scrpriv = screen->driver;
     Bool wasEnabled = pScreenPriv->enabled;
+    Bool wasMapped;
+    Bool isMapped;
     msScrPriv oldscr;
     PixmapPtr rootPixmap;
 
@@ -109,10 +112,21 @@ msSetScreenBo(ScreenPtr pScreen, struct gbm_bo *bo, Bool flip)
 
     oldscr = *scrpriv;
 
+    wasMapped = !!gbm_bo_get_map(scrpriv->front);
+    isMapped = !!gbm_bo_get_map(bo);
+
     msUnmapFramebuffer(screen);
 
     scrpriv->front = bo;
 
+    if (isMapped && !wasMapped) {
+#ifdef GLAMOR
+        glamor_egl_untexture_pixmap(rootPixmap, GLAMOR_MEMORY);
+#else
+        goto bail;
+#endif
+    }
+
     if (!msMapFramebuffer(screen)) {
         goto bail;
     }
@@ -141,7 +155,7 @@ msSetScreenBo(ScreenPtr pScreen, struct gbm_bo *bo, Bool flip)
     KdSetSubpixelOrder(pScreen, scrpriv->randr);
 
     /* Texture the front if needed */
-    if (!flip && !gbm_bo_get_map(bo)) {
+    if (!isMapped) {
 #ifdef GLAMOR
         Bool used_modifiers = gbm_bo_get_used_modifiers(bo);
         if (screen->dumb ||
@@ -183,6 +197,14 @@ bail:
                                     screen->fb.bitsPerPixel,
                                     screen->fb.byteStride,
                                     screen->fb.frameBuffer);
+
+#ifdef GLAMOR
+    if (isMapped && !wasMapped && !screen->dumb) {
+        Bool used_modifiers = gbm_bo_get_used_modifiers(scrpriv->front);
+        glamor_egl_create_textured_pixmap_from_gbm_bo(rootPixmap, scrpriv->front, used_modifiers);
+    }
+#endif
+
     if (wasEnabled) {
         KdEnableScreen(pScreen);
     }