Bug 1827615 - [Wayland] Fix Level3 and Level5 mappings. r=masayuki

libxkbcommon actually uses the strings "LevelThree" and "LevelFive" for
Level3 / Level5 respectively. To prevent this mixup from happening again,
use the defines from libxkbcommon and add fallbacks on our side as well.

Furthermore, GDK does not support Level3 / Level5 vmods in GdkModifierType,
and we also can't expect bitwise compatibility between GdkModifierType and
the opaque `(1 << xkb_keymap_mod_get_index(...))` result. Instead, use the
libxkbcommon API to reverse the vmod->rmod mapping GDK does internally, and
match against that.

See also: https://github.com/xkbcommon/libxkbcommon/discussions/732

Differential Revision: https://phabricator.services.mozilla.com/D246866

widget/gtk/mozwayland/mozwayland.c

@@ -271,3 +271,22 @@ MOZ_EXPORT int xkb_keymap_key_repeats(struct xkb_keymap* keymap,
                                       xkb_keycode_t kc) {
   return 0;
 }
+
+MOZ_EXPORT struct xkb_state* xkb_state_new(struct xkb_keymap* keymap) {
+  return NULL;
+}
+
+MOZ_EXPORT void xkb_state_unref(struct xkb_state* state) {}
+
+MOZ_EXPORT enum xkb_state_component xkb_state_update_mask(
+    struct xkb_state* state, xkb_mod_mask_t depressed_mods,
+    xkb_mod_mask_t latched_mods, xkb_mod_mask_t locked_mods,
+    xkb_layout_index_t depressed_layout, xkb_layout_index_t latched_layout,
+    xkb_layout_index_t locked_layout) {
+  return 0;
+}
+
+MOZ_EXPORT xkb_mod_mask_t xkb_state_serialize_mods(
+    struct xkb_state* state, enum xkb_state_component components) {
+  return 0;
+}

widget/gtk/nsGtkKeyUtils.cpp

@@ -651,23 +651,44 @@ void KeymapWrapper::SetModifierMask(xkb_keymap* aKeymap,
                                     ModifierIndex aModifierIndex,
                                     const char* aModifierName) {
   xkb_mod_index_t index = xkb_keymap_mod_get_index(aKeymap, aModifierName);
-  if (index != XKB_MOD_INVALID) {
-    mModifierMasks[aModifierIndex] = (1 << index);
+  if (index == XKB_MOD_INVALID) {
+    return;
+  }
+  struct xkb_state* xkb_state = xkb_state_new(aKeymap);
+  if (!xkb_state) {
+    return;
+  }
+  xkb_mod_mask_t mask = 1u << index;
+  xkb_state_update_mask(xkb_state, mask, 0, 0, 0, 0, 0);
+  xkb_mod_mask_t res =
+      xkb_state_serialize_mods(xkb_state, XKB_STATE_MODS_EFFECTIVE);
+  xkb_state_unref(xkb_state);
+  if (res == mask) {
+    // Either this already was an rmod, or it's an unmapped vmod. In the former
+    // case we obviously want to use it, in the latter this allows us to handle
+    // synthesized key events using that vmod, even if that vmod cannot be
+    // generated by keyboard (with a slight risk of GdkModifierType collisions).
+    mModifierMasks[aModifierIndex] = res;
+  } else {
+    // We found a mapped rmod for the vmod. Make sure to remove the opaque `1u
+    // << index` mask, which does not get removed by xkb_state_serialize_mods,
+    // because it can quite easily collide with the bitwise-incompatible parts
+    // of the GdkModifierType enum (most likely GDK_BUTTON{N}_MASK).
+    mModifierMasks[aModifierIndex] = res & ~(mask);
   }
 }
 
 void KeymapWrapper::SetModifierMasks(xkb_keymap* aKeymap) {
   KeymapWrapper* keymapWrapper = GetInstance();
 
-  // This mapping is derived from get_xkb_modifiers() at gdkkeys-wayland.c
-  keymapWrapper->SetModifierMask(aKeymap, INDEX_NUM_LOCK, XKB_MOD_NAME_NUM);
   keymapWrapper->SetModifierMask(aKeymap, INDEX_ALT, XKB_MOD_NAME_ALT);
-  keymapWrapper->SetModifierMask(aKeymap, INDEX_META, "Meta");
-  keymapWrapper->SetModifierMask(aKeymap, INDEX_HYPER, "Hyper");
-
-  keymapWrapper->SetModifierMask(aKeymap, INDEX_SCROLL_LOCK, "ScrollLock");
-  keymapWrapper->SetModifierMask(aKeymap, INDEX_LEVEL3, "Level3");
-  keymapWrapper->SetModifierMask(aKeymap, INDEX_LEVEL5, "Level5");
+  keymapWrapper->SetModifierMask(aKeymap, INDEX_NUM_LOCK, XKB_MOD_NAME_NUM);
+  keymapWrapper->SetModifierMask(aKeymap, INDEX_LEVEL3, XKB_VMOD_NAME_LEVEL3);
+  keymapWrapper->SetModifierMask(aKeymap, INDEX_LEVEL5, XKB_VMOD_NAME_LEVEL5);
+  keymapWrapper->SetModifierMask(aKeymap, INDEX_META, XKB_VMOD_NAME_META);
+  keymapWrapper->SetModifierMask(aKeymap, INDEX_HYPER, XKB_VMOD_NAME_HYPER);
+  keymapWrapper->SetModifierMask(aKeymap, INDEX_SCROLL_LOCK,
+                                 XKB_VMOD_NAME_SCROLL);
 
   keymapWrapper->SetKeymap(aKeymap);
 
@@ -2658,7 +2679,8 @@ void KeymapWrapper::WillDispatchKeyboardEventInternal(
   guint baseState = aGdkKeyEvent->state &
                     ~(GetGdkModifierMask(SHIFT) | GetGdkModifierMask(CTRL) |
                       GetGdkModifierMask(ALT) | GetGdkModifierMask(META) |
-                      GetGdkModifierMask(SUPER) | GetGdkModifierMask(HYPER));
+                      GetGdkModifierMask(SUPER) | GetGdkModifierMask(HYPER) |
+                      GDK_META_MASK | GDK_SUPER_MASK | GDK_HYPER_MASK);
 
   // We shold send both shifted char and unshifted char, all keyboard layout
   // users can use all keys.  Don't change event.mCharCode. On some keyboard

widget/gtk/nsGtkKeyUtils.h

@@ -19,6 +19,30 @@
 #ifdef MOZ_WAYLAND
 #  include <gdk/gdkwayland.h>
 #  include <xkbcommon/xkbcommon.h>
+#  ifndef XKB_VMOD_NAME_ALT
+#    define XKB_VMOD_NAME_ALT "Alt"
+#  endif
+#  ifndef XKB_VMOD_NAME_HYPER
+#    define XKB_VMOD_NAME_HYPER "Hyper"
+#  endif
+#  ifndef XKB_VMOD_NAME_LEVEL3
+#    define XKB_VMOD_NAME_LEVEL3 "LevelThree"
+#  endif
+#  ifndef XKB_VMOD_NAME_LEVEL5
+#    define XKB_VMOD_NAME_LEVEL5 "LevelFive"
+#  endif
+#  ifndef XKB_VMOD_NAME_META
+#    define XKB_VMOD_NAME_META "Meta"
+#  endif
+#  ifndef XKB_VMOD_NAME_NUM
+#    define XKB_VMOD_NAME_NUM "NumLock"
+#  endif
+#  ifndef XKB_VMOD_NAME_SCROLL
+#    define XKB_VMOD_NAME_SCROLL "ScrollLock"
+#  endif
+#  ifndef XKB_VMOD_NAME_SUPER
+#    define XKB_VMOD_NAME_SUPER "Super"
+#  endif
 #endif
 #include "X11UndefineNone.h"