diff --git a/modifier_only_shortcuts.cpp b/modifier_only_shortcuts.cpp
index e4fd50644f..cb47df4471 100644
--- a/modifier_only_shortcuts.cpp
+++ b/modifier_only_shortcuts.cpp
@@ -45,8 +45,9 @@ void ModifierOnlyShortcuts::keyEvent(KeyEvent *event)
         return;
     }
     if (event->type() == QEvent::KeyPress) {
-        m_pressCount++;
-        if (m_pressCount == 1 &&
+        const bool wasEmpty = m_pressedKeys.isEmpty();
+        m_pressedKeys.insert(event->nativeScanCode());
+        if (wasEmpty && m_pressedKeys.size() == 1 &&
             !ScreenLockerWatcher::self()->isLocked() &&
             m_buttonPressCount == 0 &&
             m_cachedMods == Qt::NoModifier) {
@@ -54,9 +55,9 @@ void ModifierOnlyShortcuts::keyEvent(KeyEvent *event)
         } else {
             m_modifier = Qt::NoModifier;
         }
-    } else {
-        m_pressCount--;
-        if (m_pressCount == 0 &&
+    } else if (!m_pressedKeys.isEmpty()) {
+        m_pressedKeys.remove(event->nativeScanCode());
+        if (m_pressedKeys.isEmpty() &&
             event->modifiersRelevantForGlobalShortcuts() == Qt::NoModifier &&
             !workspace()->globalShortcutsDisabled()) {
             if (m_modifier != Qt::NoModifier) {
@@ -73,6 +74,8 @@ void ModifierOnlyShortcuts::keyEvent(KeyEvent *event)
             }
         }
         m_modifier = Qt::NoModifier;
+    } else {
+        m_modifier = Qt::NoModifier;
     }
     m_cachedMods = event->modifiersRelevantForGlobalShortcuts();
 }
diff --git a/modifier_only_shortcuts.h b/modifier_only_shortcuts.h
index b75f037261..f4b7446ce7 100644
--- a/modifier_only_shortcuts.h
+++ b/modifier_only_shortcuts.h
@@ -24,6 +24,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #include <kwin_export.h>
 
 #include <QObject>
+#include <QSet>
 
 namespace KWin
 {
@@ -44,10 +45,10 @@ public:
     }
 
 private:
-    uint m_pressCount = 0;
     Qt::KeyboardModifier m_modifier = Qt::NoModifier;
     Qt::KeyboardModifiers m_cachedMods;
     uint m_buttonPressCount = 0;
+    QSet<quint32> m_pressedKeys;
 };
 
 }