Bug 1827615 - [Wayland] Fix Level3 and Level5 mappings. r=stransky

libxkbcommon actually uses the strings "LevelThree" and "LevelFive" for
Level3 / Level5 respectively. To prevent this mixup from happening again,
use the defines from libxkbcommon and add fallbacks on our side as well.

Furthermore, GDK does not support Level3 / Level5 vmods in GdkModifierType,
and we also can't expect bitwise compatibility between GdkModifierType and
the opaque `(1 << xkb_keymap_mod_get_index(...))` result. Instead, use the
libxkbcommon API to reverse the vmod->rmod mapping GDK does internally,
and match against that. Do no translate the META, SUPER and HYPER vmods,
which are natively supported by GDK, because they otherwise collide with
rmods like ALT, which we don't want to happen.

See also: xkbcommon/libxkbcommon#732

Differential Revision: https://phabricator.services.mozilla.com/D250529

Author: johnp

widget/gtk/mozwayland/mozwayland.c

@@ -271,3 +271,22 @@ MOZ_EXPORT int xkb_keymap_key_repeats(struct xkb_keymap* keymap,
                                       xkb_keycode_t kc) {
   return 0;
 }
+
+MOZ_EXPORT struct xkb_state* xkb_state_new(struct xkb_keymap* keymap) {
+  return NULL;
+}
+
+MOZ_EXPORT void xkb_state_unref(struct xkb_state* state) {}
+
+MOZ_EXPORT enum xkb_state_component xkb_state_update_mask(
+    struct xkb_state* state, xkb_mod_mask_t depressed_mods,
+    xkb_mod_mask_t latched_mods, xkb_mod_mask_t locked_mods,
+    xkb_layout_index_t depressed_layout, xkb_layout_index_t latched_layout,
+    xkb_layout_index_t locked_layout) {
+  return 0;
+}
+
+MOZ_EXPORT xkb_mod_mask_t xkb_state_serialize_mods(
+    struct xkb_state* state, enum xkb_state_component components) {
+  return 0;
+}

widget/gtk/nsGtkKeyUtils.cpp

@@ -651,23 +651,48 @@ void KeymapWrapper::SetModifierMask(xkb_keymap* aKeymap,
                                     ModifierIndex aModifierIndex,
                                     const char* aModifierName) {
   xkb_mod_index_t index = xkb_keymap_mod_get_index(aKeymap, aModifierName);
-  if (index != XKB_MOD_INVALID) {
-    mModifierMasks[aModifierIndex] = (1 << index);
+  if (index == XKB_MOD_INVALID) {
+    return;
+  }
+  struct xkb_state* xkb_state = xkb_state_new(aKeymap);
+  if (!xkb_state) {
+    return;
+  }
+  xkb_mod_mask_t mask = 1u << index;
+  xkb_state_update_mask(xkb_state, mask, 0, 0, 0, 0, 0);
+  xkb_mod_mask_t res =
+      xkb_state_serialize_mods(xkb_state, XKB_STATE_MODS_EFFECTIVE);
+  xkb_state_unref(xkb_state);
+  if (res == mask) {
+    // Either this already was an rmod, or it's an unmapped vmod. In the former
+    // case we obviously want to use it, in the latter this allows us to handle
+    // synthesized key events using that vmod, even if that vmod cannot be
+    // generated by keyboard (with a slight risk of GdkModifierType collisions).
+    mModifierMasks[aModifierIndex] = res;
+  } else {
+    // We found a mapped rmod for the vmod. Make sure to remove the opaque `1u
+    // << index` mask, which does not get removed by xkb_state_serialize_mods,
+    // because it can quite easily collide with the bitwise-incompatible parts
+    // of the GdkModifierType enum (most likely GDK_BUTTON{N}_MASK).
+    mModifierMasks[aModifierIndex] = res & ~(mask);
   }
 }
 
 void KeymapWrapper::SetModifierMasks(xkb_keymap* aKeymap) {
   KeymapWrapper* keymapWrapper = GetInstance();
 
-  // This mapping is derived from get_xkb_modifiers() at gdkkeys-wayland.c
-  keymapWrapper->SetModifierMask(aKeymap, INDEX_NUM_LOCK, XKB_MOD_NAME_NUM);
+  // These mappings are derived from get_xkb_modifiers() at gdkkeys-wayland.c,
+  // as well as libxkbcommon. The first two are practically fixed.
   keymapWrapper->SetModifierMask(aKeymap, INDEX_ALT, XKB_MOD_NAME_ALT);
-  keymapWrapper->SetModifierMask(aKeymap, INDEX_META, "Meta");
-  keymapWrapper->SetModifierMask(aKeymap, INDEX_HYPER, "Hyper");
-
-  keymapWrapper->SetModifierMask(aKeymap, INDEX_SCROLL_LOCK, "ScrollLock");
-  keymapWrapper->SetModifierMask(aKeymap, INDEX_LEVEL3, "Level3");
-  keymapWrapper->SetModifierMask(aKeymap, INDEX_LEVEL5, "Level5");
+  keymapWrapper->SetModifierMask(aKeymap, INDEX_NUM_LOCK, XKB_MOD_NAME_NUM);
+  // We have to find the rmods for these vmods, which are not supported by GDK.
+  keymapWrapper->SetModifierMask(aKeymap, INDEX_LEVEL3, XKB_VMOD_NAME_LEVEL3);
+  keymapWrapper->SetModifierMask(aKeymap, INDEX_LEVEL5, XKB_VMOD_NAME_LEVEL5);
+  keymapWrapper->SetModifierMask(aKeymap, INDEX_SCROLL_LOCK,
+                                 XKB_VMOD_NAME_SCROLL);
+  // GDK specific vmods we don't want to map to rmods to preserve behavior.
+  keymapWrapper->mModifierMasks[INDEX_HYPER] = GDK_HYPER_MASK;
+  keymapWrapper->mModifierMasks[INDEX_META] = GDK_META_MASK;
 
   keymapWrapper->SetKeymap(aKeymap);
 
@@ -2658,7 +2683,8 @@ void KeymapWrapper::WillDispatchKeyboardEventInternal(
   guint baseState = aGdkKeyEvent->state &
                     ~(GetGdkModifierMask(SHIFT) | GetGdkModifierMask(CTRL) |
                       GetGdkModifierMask(ALT) | GetGdkModifierMask(META) |
-                      GetGdkModifierMask(SUPER) | GetGdkModifierMask(HYPER));
+                      GetGdkModifierMask(SUPER) | GetGdkModifierMask(HYPER) |
+                      GDK_META_MASK | GDK_SUPER_MASK | GDK_HYPER_MASK);
 
   // We shold send both shifted char and unshifted char, all keyboard layout
   // users can use all keys.  Don't change event.mCharCode. On some keyboard

widget/gtk/nsGtkKeyUtils.h

@@ -19,6 +19,30 @@
 #ifdef MOZ_WAYLAND
 #  include <gdk/gdkwayland.h>
 #  include <xkbcommon/xkbcommon.h>
+#  ifndef XKB_VMOD_NAME_ALT
+#    define XKB_VMOD_NAME_ALT "Alt"
+#  endif
+#  ifndef XKB_VMOD_NAME_HYPER
+#    define XKB_VMOD_NAME_HYPER "Hyper"
+#  endif
+#  ifndef XKB_VMOD_NAME_LEVEL3
+#    define XKB_VMOD_NAME_LEVEL3 "LevelThree"
+#  endif
+#  ifndef XKB_VMOD_NAME_LEVEL5
+#    define XKB_VMOD_NAME_LEVEL5 "LevelFive"
+#  endif
+#  ifndef XKB_VMOD_NAME_META
+#    define XKB_VMOD_NAME_META "Meta"
+#  endif
+#  ifndef XKB_VMOD_NAME_NUM
+#    define XKB_VMOD_NAME_NUM "NumLock"
+#  endif
+#  ifndef XKB_VMOD_NAME_SCROLL
+#    define XKB_VMOD_NAME_SCROLL "ScrollLock"
+#  endif
+#  ifndef XKB_VMOD_NAME_SUPER
+#    define XKB_VMOD_NAME_SUPER "Super"
+#  endif
 #endif
 #include "X11UndefineNone.h"