Bug 1827615 - [Wayland] Fix Level3 and Level5 mappings. r=stransky

libxkbcommon actually uses the strings "LevelThree" and "LevelFive" for
Level3 / Level5 respectively. To prevent this mixup from happening again,
use the defines from libxkbcommon and add fallbacks on our side as well.

Furthermore, GDK does not support Level3 / Level5 vmods in GdkModifierType,
and we also can't expect bitwise compatibility between GdkModifierType and
the opaque `(1 << xkb_keymap_mod_get_index(...))` result. Instead, use the
libxkbcommon API to reverse the vmod->rmod mapping GDK does internally,
and match against that. Do no translate the META, SUPER and HYPER vmods,
which are natively supported by GDK, because they otherwise collide with
rmods like ALT, which we don't want to happen.

See also: xkbcommon/libxkbcommon#732

Differential Revision: https://phabricator.services.mozilla.com/D250529

UltraBlame original commit: 4ba0815eadf79c74b971d19a6f057be121ef5626

Author: marco-c

widget/gtk/mozwayland/mozwayland.c

@@ -271,3 +271,22 @@ MOZ_EXPORT int xkb_keymap_key_repeats(struct xkb_keymap* keymap,
                                       xkb_keycode_t kc) {
   return 0;
 }
+
+MOZ_EXPORT struct xkb_state* xkb_state_new(struct xkb_keymap* keymap) {
+  return NULL;
+}
+
+MOZ_EXPORT void xkb_state_unref(struct xkb_state* state) {}
+
+MOZ_EXPORT enum xkb_state_component xkb_state_update_mask(
+    struct xkb_state* state, xkb_mod_mask_t depressed_mods,
+    xkb_mod_mask_t latched_mods, xkb_mod_mask_t locked_mods,
+    xkb_layout_index_t depressed_layout, xkb_layout_index_t latched_layout,
+    xkb_layout_index_t locked_layout) {
+  return 0;
+}
+
+MOZ_EXPORT xkb_mod_mask_t xkb_state_serialize_mods(
+    struct xkb_state* state, enum xkb_state_component components) {
+  return 0;
+}

widget/gtk/nsGtkKeyUtils.cpp

@@ -651,23 +651,48 @@ void KeymapWrapper::SetModifierMask(xkb_keymap* aKeymap,
                                     ModifierIndex aModifierIndex,
                                     const char* aModifierName) {
   xkb_mod_index_t index = xkb_keymap_mod_get_index(aKeymap, aModifierName);
-  if (index != XKB_MOD_INVALID) {
-    mModifierMasks[aModifierIndex] = (1 << index);
+  if (index == XKB_MOD_INVALID) {
+    return;
+  }
+  struct xkb_state* xkb_state = xkb_state_new(aKeymap);
+  if (!xkb_state) {
+    return;
+  }
+  xkb_mod_mask_t mask = 1u << index;
+  xkb_state_update_mask(xkb_state, mask, 0, 0, 0, 0, 0);
+  xkb_mod_mask_t res =
+      xkb_state_serialize_mods(xkb_state, XKB_STATE_MODS_EFFECTIVE);
+  xkb_state_unref(xkb_state);
+  if (res == mask) {
+    
+    
+    
+    
+    mModifierMasks[aModifierIndex] = res;
+  } else {
+    
+    
+    
+    
+    mModifierMasks[aModifierIndex] = res & ~(mask);
   }
 }
 
 void KeymapWrapper::SetModifierMasks(xkb_keymap* aKeymap) {
   KeymapWrapper* keymapWrapper = GetInstance();
 
 
-  keymapWrapper->SetModifierMask(aKeymap, INDEX_NUM_LOCK, XKB_MOD_NAME_NUM);
+  
   keymapWrapper->SetModifierMask(aKeymap, INDEX_ALT, XKB_MOD_NAME_ALT);
-  keymapWrapper->SetModifierMask(aKeymap, INDEX_META, "Meta");
-  keymapWrapper->SetModifierMask(aKeymap, INDEX_HYPER, "Hyper");
-
-  keymapWrapper->SetModifierMask(aKeymap, INDEX_SCROLL_LOCK, "ScrollLock");
-  keymapWrapper->SetModifierMask(aKeymap, INDEX_LEVEL3, "Level3");
-  keymapWrapper->SetModifierMask(aKeymap, INDEX_LEVEL5, "Level5");
+  keymapWrapper->SetModifierMask(aKeymap, INDEX_NUM_LOCK, XKB_MOD_NAME_NUM);
+  
+  keymapWrapper->SetModifierMask(aKeymap, INDEX_LEVEL3, XKB_VMOD_NAME_LEVEL3);
+  keymapWrapper->SetModifierMask(aKeymap, INDEX_LEVEL5, XKB_VMOD_NAME_LEVEL5);
+  keymapWrapper->SetModifierMask(aKeymap, INDEX_SCROLL_LOCK,
+                                 XKB_VMOD_NAME_SCROLL);
+  
+  keymapWrapper->mModifierMasks[INDEX_HYPER] = GDK_HYPER_MASK;
+  keymapWrapper->mModifierMasks[INDEX_META] = GDK_META_MASK;
 
   keymapWrapper->SetKeymap(aKeymap);
 
@@ -2658,7 +2683,8 @@ void KeymapWrapper::WillDispatchKeyboardEventInternal(
   guint baseState = aGdkKeyEvent->state &
                     ~(GetGdkModifierMask(SHIFT) | GetGdkModifierMask(CTRL) |
                       GetGdkModifierMask(ALT) | GetGdkModifierMask(META) |
-                      GetGdkModifierMask(SUPER) | GetGdkModifierMask(HYPER));
+                      GetGdkModifierMask(SUPER) | GetGdkModifierMask(HYPER) |
+                      GDK_META_MASK | GDK_SUPER_MASK | GDK_HYPER_MASK);
 
 
 

widget/gtk/nsGtkKeyUtils.h

@@ -19,6 +19,30 @@
 #ifdef MOZ_WAYLAND
 #  include <gdk/gdkwayland.h>
 #  include <xkbcommon/xkbcommon.h>
+#  ifndef XKB_VMOD_NAME_ALT
+#    define XKB_VMOD_NAME_ALT "Alt"
+#  endif
+#  ifndef XKB_VMOD_NAME_HYPER
+#    define XKB_VMOD_NAME_HYPER "Hyper"
+#  endif
+#  ifndef XKB_VMOD_NAME_LEVEL3
+#    define XKB_VMOD_NAME_LEVEL3 "LevelThree"
+#  endif
+#  ifndef XKB_VMOD_NAME_LEVEL5
+#    define XKB_VMOD_NAME_LEVEL5 "LevelFive"
+#  endif
+#  ifndef XKB_VMOD_NAME_META
+#    define XKB_VMOD_NAME_META "Meta"
+#  endif
+#  ifndef XKB_VMOD_NAME_NUM
+#    define XKB_VMOD_NAME_NUM "NumLock"
+#  endif
+#  ifndef XKB_VMOD_NAME_SCROLL
+#    define XKB_VMOD_NAME_SCROLL "ScrollLock"
+#  endif
+#  ifndef XKB_VMOD_NAME_SUPER
+#    define XKB_VMOD_NAME_SUPER "Super"
+#  endif
 #endif
 #include "X11UndefineNone.h"