8335468: [XWayland] JavaFX hangs when calling java.awt.Robot.getPixelColor

Backport-of: 965aace297154ab08ee41a4d988553707cae8b32

Author: GoeLin

src/java.desktop/unix/native/libawt_xawt/awt/fp_pipewire.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2023, 2024, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -58,7 +58,6 @@ void (*fp_pw_stream_destroy)(struct pw_stream *stream);
 
 
 void (*fp_pw_init)(int *argc, char **argv[]);
-void (*fp_pw_deinit)(void);
 
 struct pw_core *
 (*fp_pw_context_connect_fd)(struct pw_context *context,

src/java.desktop/unix/native/libawt_xawt/awt/gtk3_interface.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2005, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2005, 2024, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -285,6 +285,9 @@ GtkApi* gtk3_load(JNIEnv *env, const char* lib_name)
 
         fp_g_main_context_iteration =
             dl_symbol("g_main_context_iteration");
+        fp_g_main_context_default = dl_symbol("g_main_context_default");
+        fp_g_main_context_is_owner = dl_symbol("g_main_context_is_owner");
+
 
         fp_g_value_init = dl_symbol("g_value_init");
         fp_g_type_is_a = dl_symbol("g_type_is_a");
@@ -556,6 +559,7 @@ GtkApi* gtk3_load(JNIEnv *env, const char* lib_name)
         fp_g_signal_connect_data = dl_symbol("g_signal_connect_data");
         fp_gtk_widget_show = dl_symbol("gtk_widget_show");
         fp_gtk_main = dl_symbol("gtk_main");
+        fp_gtk_main_level = dl_symbol("gtk_main_level");
 
         fp_g_path_get_dirname = dl_symbol("g_path_get_dirname");
 
@@ -3122,6 +3126,8 @@ static void gtk3_init(GtkApi* gtk) {
     gtk->g_uuid_string_is_valid = fp_g_uuid_string_is_valid;
 
     gtk->g_main_context_iteration = fp_g_main_context_iteration;
+    gtk->g_main_context_default = fp_g_main_context_default;
+    gtk->g_main_context_is_owner = fp_g_main_context_is_owner;
     gtk->g_error_free = fp_g_error_free;
     gtk->g_unix_fd_list_get = fp_g_unix_fd_list_get;
 

src/java.desktop/unix/native/libawt_xawt/awt/gtk3_interface.h

@@ -392,6 +392,9 @@ static void         (*fp_g_object_set)(gpointer object,
                                        ...);
 
 static gboolean (*fp_g_main_context_iteration)(GMainContext *context, gboolean may_block);
+static GMainContext *(*fp_g_main_context_default)();
+static gboolean (*fp_g_main_context_is_owner)(GMainContext* context);
+
 static gboolean (*fp_g_str_has_prefix)(const gchar *str, const gchar *prefix);
 static gchar** (*fp_g_strsplit)(const gchar *string, const gchar *delimiter,
            gint max_tokens);

src/java.desktop/unix/native/libawt_xawt/awt/gtk_interface.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2005, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2005, 2024, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -792,6 +792,8 @@ typedef struct GtkApi {
 
     gboolean (*g_main_context_iteration)(GMainContext *context,
                                          gboolean may_block);
+    GMainContext *(*g_main_context_default)();
+    gboolean (*g_main_context_is_owner)(GMainContext* context);
 
     void (*g_error_free)(GError *error);
 

src/java.desktop/unix/native/libawt_xawt/awt/screencast_pipewire.c

@@ -49,6 +49,7 @@ static GString *activeSessionToken;
 
 struct ScreenSpace screenSpace = {0};
 static struct PwLoopData pw = {0};
+volatile bool isGtkMainThread = FALSE;
 
 jclass tokenStorageClass = NULL;
 jmethodID storeTokenMethodID = NULL;
@@ -132,10 +133,6 @@ static void doCleanup() {
         screenSpace.screenCount = 0;
     }
 
-    if (!sessionClosed) {
-        fp_pw_deinit();
-    }
-
     gtk->g_string_set_size(activeSessionToken, 0);
     sessionClosed = TRUE;
 }
@@ -581,6 +578,13 @@ static gboolean doLoop(GdkRectangle requestedArea) {
     if (!pw.loop && !sessionClosed) {
         pw.loop = fp_pw_thread_loop_new("AWT Pipewire Thread", NULL);
 
+        if (!pw.loop) {
+            // in case someone called the pw_deinit before
+            DEBUG_SCREENCAST("pw_init\n", NULL);
+            fp_pw_init(NULL, NULL);
+            pw.loop = fp_pw_thread_loop_new("AWT Pipewire Thread", NULL);
+        }
+
         if (!pw.loop) {
             DEBUG_SCREENCAST("!!! Could not create a loop\n", NULL);
             doCleanup();
@@ -712,7 +716,6 @@ static gboolean loadSymbols() {
     LOAD_SYMBOL(fp_pw_stream_disconnect, "pw_stream_disconnect");
     LOAD_SYMBOL(fp_pw_stream_destroy, "pw_stream_destroy");
     LOAD_SYMBOL(fp_pw_init, "pw_init");
-    LOAD_SYMBOL(fp_pw_deinit, "pw_deinit");
     LOAD_SYMBOL(fp_pw_context_connect_fd, "pw_context_connect_fd");
     LOAD_SYMBOL(fp_pw_core_disconnect, "pw_core_disconnect");
     LOAD_SYMBOL(fp_pw_context_new, "pw_context_new");
@@ -946,9 +949,10 @@ JNIEXPORT jint JNICALL Java_sun_awt_screencast_ScreencastHelper_getRGBPixelsImpl
                          ? (*env)->GetStringUTFChars(env, jtoken, NULL)
                          : NULL;
 
+    isGtkMainThread = gtk->g_main_context_is_owner(gtk->g_main_context_default());
     DEBUG_SCREENCAST(
-            "taking screenshot at \n\tx: %5i y %5i w %5i h %5i with token |%s|\n",
-            jx, jy, jwidth, jheight, token
+            "taking screenshot at \n\tx: %5i y %5i w %5i h %5i\n\twith token |%s| isGtkMainThread %d\n",
+            jx, jy, jwidth, jheight, token, isGtkMainThread
     );
 
     int attemptResult = makeScreencast(

src/java.desktop/unix/native/libawt_xawt/awt/screencast_portal.c

@@ -32,6 +32,7 @@
 #include "screencast_pipewire.h"
 #include "screencast_portal.h"
 
+extern volatile bool isGtkMainThread;
 
 extern struct ScreenSpace screenSpace;
 
@@ -67,6 +68,27 @@ gboolean validateToken(const gchar *token) {
     return isValid;
 }
 
+void waitForCallback(struct DBusCallbackHelper *helper) {
+    if (!helper) {
+        return;
+    }
+
+    if (isGtkMainThread) {
+        gtk->gtk_main();
+    } else {
+        while (!helper->isDone) {
+            // do not block if there is a GTK loop running
+            gtk->g_main_context_iteration(NULL, gtk->gtk_main_level() == 0);
+        }
+    }
+}
+
+void callbackEnd() {
+    if (isGtkMainThread) {
+        gtk->gtk_main_quit();
+    }
+}
+
 /**
  * @return TRUE on success
  */
@@ -362,6 +384,7 @@ static void callbackScreenCastCreateSession(
     }
 
     helper->isDone = TRUE;
+    callbackEnd();
 }
 
 gboolean portalScreenCastCreateSession() {
@@ -426,9 +449,7 @@ gboolean portalScreenCastCreateSession() {
                          err->message);
         ERR_HANDLE(err);
     } else {
-        while (!helper.isDone) {
-            gtk->g_main_context_iteration(NULL, TRUE);
-        }
+        waitForCallback(&helper);
     }
 
     unregisterScreenCastCallback(&helper);
@@ -472,6 +493,8 @@ static void callbackScreenCastSelectSources(
     if (result) {
         gtk->g_variant_unref(result);
     }
+
+    callbackEnd();
 }
 
 gboolean portalScreenCastSelectSources(const gchar *token) {
@@ -552,9 +575,7 @@ gboolean portalScreenCastSelectSources(const gchar *token) {
         DEBUG_SCREENCAST("Failed to call SelectSources: %s\n", err->message);
         ERR_HANDLE(err);
     } else {
-        while (!helper.isDone) {
-            gtk->g_main_context_iteration(NULL, TRUE);
-        }
+        waitForCallback(&helper);
     }
 
     unregisterScreenCastCallback(&helper);
@@ -640,6 +661,8 @@ static void callbackScreenCastStart(
     if (streams) {
         gtk->g_variant_unref(streams);
     }
+
+    callbackEnd();
 }
 
 ScreenCastResult portalScreenCastStart(const gchar *token) {
@@ -693,9 +716,7 @@ ScreenCastResult portalScreenCastStart(const gchar *token) {
         DEBUG_SCREENCAST("Failed to start session: %s\n", err->message);
         ERR_HANDLE(err);
     } else {
-        while (!helper.isDone) {
-            gtk->g_main_context_iteration(NULL, TRUE);
-        }
+        waitForCallback(&helper);
     }
 
     unregisterScreenCastCallback(&helper);