diff --git a/src/modifier_only_shortcuts.cpp b/src/modifier_only_shortcuts.cpp
index c68c1eb95a..47da93b1e6 100644
--- a/src/modifier_only_shortcuts.cpp
+++ b/src/modifier_only_shortcuts.cpp
@@ -39,7 +39,7 @@ void ModifierOnlyShortcuts::keyEvent(KeyEvent *event)
         m_pressedKeys.insert(event->nativeScanCode());
         if (wasEmpty && m_pressedKeys.size() == 1 &&
             !ScreenLockerWatcher::self()->isLocked() &&
-            m_buttonPressCount == 0 &&
+            m_pressedButtons == Qt::NoButton &&
             m_cachedMods == Qt::NoModifier) {
             m_modifier = Qt::KeyboardModifier(int(event->modifiersRelevantForGlobalShortcuts()));
         } else {
@@ -77,11 +77,7 @@ void ModifierOnlyShortcuts::pointerEvent(MouseEvent *event)
     if (event->type() == QEvent::MouseMove) {
         return;
     }
-    if (event->type() == QEvent::MouseButtonPress) {
-        m_buttonPressCount++;
-    } else if (event->type() == QEvent::MouseButtonRelease) {
-        m_buttonPressCount--;
-    }
+    m_pressedButtons = event->buttons();
     reset();
 }
 
diff --git a/src/modifier_only_shortcuts.h b/src/modifier_only_shortcuts.h
index 58a268e21b..ef0bbfc15b 100644
--- a/src/modifier_only_shortcuts.h
+++ b/src/modifier_only_shortcuts.h
@@ -36,7 +36,7 @@ public:
 private:
     Qt::KeyboardModifier m_modifier = Qt::NoModifier;
     Qt::KeyboardModifiers m_cachedMods;
-    uint m_buttonPressCount = 0;
+    Qt::MouseButtons m_pressedButtons;
     QSet<quint32> m_pressedKeys;
 };