always force zero rotation in KMS planes for consistency

primary planes are sometimes rotated when the framebuffer console is
rotated. While it is potentially useful to respect that, and rotate the
app accordingly, it won't rotate the input of the app nor the cursor
plane.

Just use zero rotation everywhere.

Provide feedback if a cursor vs. overlay plane was allocated, if
prefer_cursor was set.

We can only use fast cursor updates if a cursor plane was allocated.

Author: ardera

src/dmabuf_surface.c

@@ -310,7 +310,8 @@ static int dmabuf_surface_present_kms(struct surface *_s, const struct fl_layer_
         },
         refcounted_dmabuf_unref_void,
         NULL,
-        refcounted_dmabuf_ref(s->next_buf)
+        refcounted_dmabuf_ref(s->next_buf),
+        NULL
     );
     if (ok != 0) {
         LOG_ERROR("Couldn't push KMS fb layer. kms_req_builder_push_fb_layer: %s\n", strerror(ok));

src/egl_gbm_render_surface.c

@@ -533,15 +533,20 @@ static int egl_gbm_render_surface_present_kms(struct surface *s, const struct fl
             .src_w = DOUBLE_TO_FP1616_ROUNDED(egl_surface->render_surface.size.x),
             .src_h = DOUBLE_TO_FP1616_ROUNDED(egl_surface->render_surface.size.y),
 
-            .has_rotation = false,
+            // If a rotated framebuffer console is shown, the rotation of the primary plane might be non-zero.
+            //
+            // Even though it'd be nice to keep using the already set rotation, other planes might not be rotated,
+            // so just use zero rotation for all planes.
+            .has_rotation = true,
             .rotation = PLANE_TRANSFORM_ROTATE_0,
 
             .has_in_fence_fd = false,
             .in_fence_fd = 0,
         },
         on_release_layer,
         NULL,
-        locked_fb_ref(egl_surface->locked_front_fb)
+        locked_fb_ref(egl_surface->locked_front_fb),
+        NULL
     );
     TRACER_END(egl_surface->surface.tracer, "kms_req_builder_push_fb_layer");
     if (ok != 0) {

src/modesetting.c

@@ -2,6 +2,7 @@
 
 #include <errno.h>
 #include <inttypes.h>
+#include <stdatomic.h>
 #include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
@@ -2400,7 +2401,8 @@ int kms_req_builder_push_fb_layer(
     const struct kms_fb_layer *layer,
     kms_fb_release_cb_t release_callback,
     kms_deferred_fb_release_cb_t deferred_release_callback,
-    void *userdata
+    void *userdata,
+    bool *allocated_cursor_plane
 ) {
     struct drm_plane *plane;
     int64_t zpos;
@@ -2449,7 +2451,10 @@ int kms_req_builder_push_fb_layer(
             // clang-format on
         );
         if (plane == NULL) {
+            if (allocated_cursor_plane) *allocated_cursor_plane = false;
             LOG_DEBUG("Couldn't find a fitting cursor plane.\n");
+        } else  {
+            if (allocated_cursor_plane) *allocated_cursor_plane = true;
         }
     }
 

src/modesetting.h

@@ -861,6 +861,10 @@ bool kms_req_builder_prefer_next_layer_opaque(struct kms_req_builder *builder);
  *                                  instead.
  * @param userdata Userdata pointer that's passed to the release_callback or
  *                 deferred_release_callback as-is.
+ * @param allocated_cursor_plane    When layer->prefer_cursor is set, this will
+ *                                  be set to true if a cursor layer was
+ *                                  successfully allocated. Otherwise, it will
+ *                                  be set to false.
  * @returns Zero on success, otherwise:
  *            - EINVAL: if attempting to push a second framebuffer layer, if
  *                driver supports atomic modesetting but legacy modesetting is
@@ -877,7 +881,8 @@ int kms_req_builder_push_fb_layer(
     const struct kms_fb_layer *layer,
     kms_fb_release_cb_t release_callback,
     kms_deferred_fb_release_cb_t deferred_release_callback,
-    void *userdata
+    void *userdata,
+    bool *allocated_cursor_plane
 );
 
 /**

src/vk_gbm_render_surface.c

@@ -687,15 +687,17 @@ static int vk_gbm_render_surface_present_kms(struct surface *s, const struct fl_
             .src_w = DOUBLE_TO_FP1616_ROUNDED(vk_surface->render_surface.size.x),
             .src_h = DOUBLE_TO_FP1616_ROUNDED(vk_surface->render_surface.size.y),
 
-            .has_rotation = false,
+            // see egl_gbm_render_surface_present_kms
+            .has_rotation = true,
             .rotation = PLANE_TRANSFORM_ROTATE_0,
 
             .has_in_fence_fd = false,
             .in_fence_fd = 0,
         },
         on_release_layer,
         NULL,
-        locked_fb_ref(vk_surface->front_fb)
+        locked_fb_ref(vk_surface->front_fb),
+        NULL
     );
     TRACER_END(vk_surface->surface.tracer, "kms_req_builder_push_fb_layer");
     if (ok != 0) {

src/window.c

@@ -173,6 +173,7 @@ struct window {
         struct cursor_buffer *cursor;
 
         bool logged_cursor_plane_allocation_failed;
+        bool has_cursor_plane;
     } kms;
 
     /**
@@ -1202,17 +1203,19 @@ static int kms_window_push_composition_locked(struct window *window, struct fl_l
                 .dst_y = window->cursor_pos.y - window->kms.cursor->hotspot.y,
                 .dst_w = window->kms.cursor->width,
                 .dst_h = window->kms.cursor->height,
-                .has_rotation = false,
-                .rotation = PLANE_TRANSFORM_NONE,
+                .has_rotation = true,
+                .rotation = PLANE_TRANSFORM_ROTATE_0,
                 .has_in_fence_fd = false,
                 .in_fence_fd = 0,
                 .prefer_cursor = true,
             },
             cursor_buffer_unref_with_locked_drmdev,
             NULL,
-            window->kms.cursor
+            window->kms.cursor,
+            &window->kms.has_cursor_plane
         );
         if (ok != 0) {
+            window->kms.has_cursor_plane = false;
             if (!window->kms.logged_cursor_plane_allocation_failed) {
                 window->kms.logged_cursor_plane_allocation_failed = true;
                 LOG_ERROR("Couldn't present cursor.\n");
@@ -1575,7 +1578,21 @@ static int kms_window_set_cursor_locked(
         } else if (has_pos) {
             // apply the new cursor position using drmModeMoveCursor
             window->cursor_pos = pos;
-            drmdev_move_cursor(window->kms.drmdev, window->kms.crtc->id, vec2i_sub(pos, window->kms.cursor->hotspot));
+
+            // if we have a hardware cursor plane, just move the cursor.
+            // this is very fast and we can do that a lot of times per frame.
+            //
+            // if we don't have a hardware cursor plane, we need to do a full commit.
+            // However, we can only do one commit per vsync, so if flutter also later
+            // wants to present a frame, we wouldn't be able to present it. Even later
+            // cursor movements in the same vsync interval we couldn't show on screen anymore.
+            //
+            // Mutter (GNOME's compositor) has an extra KMS cursor thread with high priority for that purpose.
+            if (window->kms.has_cursor_plane) {
+                drmdev_move_cursor(window->kms.drmdev, window->kms.crtc->id, vec2i_sub(pos, window->kms.cursor->hotspot));
+            } else {
+                // for now just don't do anything.
+            }
         }
     } else {
         if (window->kms.cursor != NULL) {