surface-actor-x11: Move window related unredirect logic to MetaWindowX11

Better to have the relevant object figure out whether it is a good
position to be unredirectable other than the actor, which should be
responsible for being composited.

Original Mutter commit: https://gitlab.gnome.org/GNOME/mutter/-/merge_requests/798/diffs?commit_id=5dad87cfb9058a0be96d9ec1a530e09a6d418a6e

Author: Sunderland93

src/compositor/meta-surface-actor-x11.c

@@ -254,33 +254,14 @@ meta_surface_actor_x11_is_opaque (MetaSurfaceActor *actor)
 gboolean
 meta_surface_actor_x11_should_unredirect (MetaSurfaceActorX11 *self)
 {
-  MetaWindow *window = self->window;
-
-  if (meta_window_requested_dont_bypass_compositor (window))
-    return FALSE;
-
-  if (window->opacity != 0xFF)
-    return FALSE;
-
-  if (window->shape_region != NULL)
-    return FALSE;
-
-  if (!meta_window_is_monitor_sized (window))
-    return FALSE;
-
-  if (meta_window_requested_bypass_compositor (window))
-    return TRUE;
-
   if (!meta_surface_actor_x11_is_opaque (META_SURFACE_ACTOR (self)))
     return FALSE;
 
-  if (meta_window_is_override_redirect (window))
-    return TRUE;
-
-  if (self->does_full_damage && meta_prefs_get_unredirect_fullscreen_windows ())
-    return TRUE;
+  if (!self->does_full_damage &&
+      !meta_window_is_override_redirect (self->window))
+    return FALSE;
 
-  return FALSE;
+  return TRUE;
 }
 
 static void

src/compositor/meta-window-actor-x11.c

@@ -39,6 +39,7 @@
 #include "meta/meta-window-actor.h"
 #include "meta/meta-x11-errors.h"
 #include "meta/window.h"
+#include "x11/window-x11.h"
 #include "x11/meta-x11-display-private.h"
 #include "x11/window-x11.h"
 
@@ -544,12 +545,18 @@ has_shadow (MetaWindowActorX11 *actor_x11)
 gboolean
 meta_window_actor_x11_should_unredirect (MetaWindowActorX11 *actor_x11)
 {
+  MetaWindow *window =
+    meta_window_actor_get_meta_window (META_WINDOW_ACTOR (actor_x11));
+  MetaWindowX11 *window_x11 = META_WINDOW_X11 (window);
   MetaSurfaceActor *surface;
   MetaSurfaceActorX11 *surface_x11;
 
   if (meta_window_actor_is_destroyed (META_WINDOW_ACTOR (actor_x11)))
     return FALSE;
 
+  if (!meta_window_x11_can_unredirect (window_x11))
+    return FALSE;
+
   surface = meta_window_actor_get_surface (META_WINDOW_ACTOR (actor_x11));
   if (!surface)
     return FALSE;

src/core/window-private.h

@@ -65,13 +65,6 @@ typedef enum
 
 #define NUMBER_OF_QUEUES 3
 
-typedef enum
-{
-  _NET_WM_BYPASS_COMPOSITOR_HINT_AUTO = 0,
-  _NET_WM_BYPASS_COMPOSITOR_HINT_ON = 1,
-  _NET_WM_BYPASS_COMPOSITOR_HINT_OFF = 2,
-} MetaBypassCompositorHintValue;
-
 typedef enum
 {
   META_MOVE_RESIZE_CONFIGURE_REQUEST = 1 << 0,
@@ -550,9 +543,6 @@ struct _MetaWindow
 
   // MetaWindow *tile_match;
 
-  /* Bypass compositor hints */
-  guint bypass_compositor;
-
   struct {
     MetaPlacementRule *rule;
     MetaPlacementState state;

src/core/window.c

@@ -3165,30 +3165,6 @@ meta_window_is_on_primary_monitor (MetaWindow *window)
   return window->monitor->is_primary;
 }
 
-/**
- * meta_window_requested_bypass_compositor:
- * @window: a #MetaWindow
- *
- * Return value: %TRUE if the window requested to bypass the compositor
- */
-gboolean
-meta_window_requested_bypass_compositor (MetaWindow *window)
-{
-  return window->bypass_compositor == _NET_WM_BYPASS_COMPOSITOR_HINT_ON;
-}
-
-/**
- * meta_window_requested_dont_bypass_compositor:
- * @window: a #MetaWindow
- *
- * Return value: %TRUE if the window requested to opt out of unredirecting
- */
-gboolean
-meta_window_requested_dont_bypass_compositor (MetaWindow *window)
-{
-  return window->bypass_compositor == _NET_WM_BYPASS_COMPOSITOR_HINT_OFF;
-}
-
 static void
 get_default_tile_fractions (MetaTileMode for_mode,
                             double *hfraction,

src/meta/window.h

@@ -283,12 +283,6 @@ gboolean          meta_window_is_monitor_sized (MetaWindow *window);
 META_EXPORT
 gboolean          meta_window_is_on_primary_monitor (MetaWindow *window);
 
-META_EXPORT
-gboolean          meta_window_requested_bypass_compositor (MetaWindow *window);
-
-META_EXPORT
-gboolean          meta_window_requested_dont_bypass_compositor (MetaWindow *window);
-
 META_EXPORT
 gboolean meta_window_get_icon_geometry (MetaWindow    *window,
                                         MetaRectangle *rect);

src/x11/window-props.c

@@ -1847,23 +1847,28 @@ reload_bypass_compositor (MetaWindow    *window,
                           MetaPropValue *value,
                           gboolean       initial)
 {
-  int requested_value = 0;
-  int current_value = window->bypass_compositor;
+  MetaWindowX11 *window_x11 = META_WINDOW_X11 (window);
+  MetaWindowX11Private *priv = meta_window_x11_get_private (window_x11);
+  MetaBypassCompositorHint requested_value;
+  MetaBypassCompositorHint current_value;
 
   if (value->type != META_PROP_VALUE_INVALID)
-      requested_value = (int) value->v.cardinal;
+    requested_value = (MetaBypassCompositorHint) value->v.cardinal;
+  else
+    requested_value = META_BYPASS_COMPOSITOR_HINT_AUTO;
 
+  current_value = priv->bypass_compositor;
   if (requested_value == current_value)
     return;
 
-  if (requested_value == _NET_WM_BYPASS_COMPOSITOR_HINT_ON)
+  if (requested_value == META_BYPASS_COMPOSITOR_HINT_ON)
     meta_verbose ("Request to bypass compositor for window %s.\n", window->desc);
-  else if (requested_value == _NET_WM_BYPASS_COMPOSITOR_HINT_OFF)
+  else if (requested_value == META_BYPASS_COMPOSITOR_HINT_OFF)
     meta_verbose ("Request to don't bypass compositor for window %s.\n", window->desc);
-  else if (requested_value != _NET_WM_BYPASS_COMPOSITOR_HINT_AUTO)
+  else if (requested_value != META_BYPASS_COMPOSITOR_HINT_AUTO)
     return;
 
-  window->bypass_compositor = requested_value;
+  priv->bypass_compositor = requested_value;
 }
 
 static void

src/x11/window-x11-private.h

@@ -29,6 +29,16 @@
 
 G_BEGIN_DECLS
 
+/*
+ * Mirrors _NET_WM_BYPASS_COMPOSITOR preference values.
+ */
+typedef enum _MetaBypassCompositorHint
+{
+  META_BYPASS_COMPOSITOR_HINT_AUTO = 0,
+  META_BYPASS_COMPOSITOR_HINT_ON = 1,
+  META_BYPASS_COMPOSITOR_HINT_OFF = 2,
+} MetaBypassCompositorHint;
+
 typedef struct _MetaWindowX11Private MetaWindowX11Private;
 
 struct _MetaWindowX11Private
@@ -64,10 +74,16 @@ struct _MetaWindowX11Private
 
   /* Freeze/thaw on resize (for Xwayland) */
   gboolean thaw_after_paint;
+
+  /* Bypass compositor hints */
+  MetaBypassCompositorHint bypass_compositor;
 };
 
 MetaWindowX11Private * meta_window_x11_get_private (MetaWindowX11 *window_x11);
 
+void meta_window_x11_set_bypass_compositor_hint (MetaWindowX11            *window_x11,
+                                                 MetaBypassCompositorHint  requested_value);
+
 G_END_DECLS
 
 #endif

src/x11/window-x11.c

@@ -4146,3 +4146,62 @@ meta_window_x11_get_client_rect (MetaWindowX11 *window_x11)
 
   return priv->client_rect;
 }
+
+static gboolean
+has_requested_bypass_compositor (MetaWindowX11 *window_x11)
+{
+  MetaWindowX11Private *priv = meta_window_x11_get_instance_private (window_x11);
+
+  return priv->bypass_compositor == META_BYPASS_COMPOSITOR_HINT_ON;
+}
+
+static gboolean
+has_requested_dont_bypass_compositor (MetaWindowX11 *window_x11)
+{
+  MetaWindowX11Private *priv = meta_window_x11_get_instance_private (window_x11);
+
+  return priv->bypass_compositor == META_BYPASS_COMPOSITOR_HINT_OFF;
+}
+
+gboolean
+meta_window_x11_can_unredirect (MetaWindowX11 *window_x11)
+{
+  MetaWindow *window = META_WINDOW (window_x11);
+
+  if (has_requested_dont_bypass_compositor (window_x11))
+    return FALSE;
+
+  if (window->opacity != 0xFF)
+    return FALSE;
+
+  if (window->shape_region != NULL)
+    return FALSE;
+
+  if (!window->monitor)
+    return FALSE;
+
+  if (window->fullscreen)
+    return TRUE;
+
+  if (meta_window_is_screen_sized (window))
+    return TRUE;
+
+  if (has_requested_bypass_compositor (window_x11))
+    return TRUE;
+
+  if (window->override_redirect)
+    {
+      MetaRectangle window_rect;
+      MetaRectangle logical_monitor_layout;
+      MetaLogicalMonitor *logical_monitor = window->monitor;
+
+      meta_window_get_frame_rect (window, &window_rect);
+      logical_monitor_layout =
+        meta_logical_monitor_get_layout (logical_monitor);
+
+      if (meta_rectangle_equal (&window_rect, &logical_monitor_layout))
+        return TRUE;
+    }
+
+  return FALSE;
+}

src/x11/window-x11.h

@@ -97,4 +97,6 @@ void meta_window_x11_surface_rect_to_client_rect (MetaWindow    *window,
                                                   MetaRectangle *client_rect);
 
 MetaRectangle meta_window_x11_get_client_rect    (MetaWindowX11 *window_x11);
+
+gboolean meta_window_x11_can_unredirect          (MetaWindowX11 *window_x11);
 #endif