diff --git a/autotests/integration/modifier_only_shortcut_test.cpp b/autotests/integration/modifier_only_shortcut_test.cpp
index b4b1c16b22..dca99aae63 100644
--- a/autotests/integration/modifier_only_shortcut_test.cpp
+++ b/autotests/integration/modifier_only_shortcut_test.cpp
@@ -197,7 +197,6 @@ void ModifierOnlyShortcutTest::testTrigger()
     kwinApp()->platform()->keyboardKeyReleased(modifier, timestamp++);
     kwinApp()->platform()->pointerButtonReleased(BTN_LEFT, timestamp++);
     QCOMPARE(input()->qtButtonStates(), Qt::NoButton);
-    QEXPECT_FAIL("", "Button not yet handled", Continue);
     QCOMPARE(triggeredSpy.count(), 2);
 
     // mouse button press before mod press, release before mod release
@@ -207,7 +206,6 @@ void ModifierOnlyShortcutTest::testTrigger()
     kwinApp()->platform()->pointerButtonReleased(BTN_LEFT, timestamp++);
     kwinApp()->platform()->keyboardKeyReleased(modifier, timestamp++);
     QCOMPARE(input()->qtButtonStates(), Qt::NoButton);
-    QEXPECT_FAIL("", "Button not yet handled", Continue);
     QCOMPARE(triggeredSpy.count(), 2);
 
     // mouse button click while mod is pressed
@@ -217,21 +215,18 @@ void ModifierOnlyShortcutTest::testTrigger()
     kwinApp()->platform()->pointerButtonReleased(BTN_LEFT, timestamp++);
     kwinApp()->platform()->keyboardKeyReleased(modifier, timestamp++);
     QCOMPARE(input()->qtButtonStates(), Qt::NoButton);
-    QEXPECT_FAIL("", "Button not yet handled", Continue);
     QCOMPARE(triggeredSpy.count(), 2);
 
     // scroll while mod is pressed
     kwinApp()->platform()->keyboardKeyPressed(modifier, timestamp++);
     kwinApp()->platform()->pointerAxisVertical(5.0, timestamp++);
     kwinApp()->platform()->keyboardKeyReleased(modifier, timestamp++);
-    QEXPECT_FAIL("", "Axis not yet handled", Continue);
     QCOMPARE(triggeredSpy.count(), 2);
 
     // same for horizontal
     kwinApp()->platform()->keyboardKeyPressed(modifier, timestamp++);
     kwinApp()->platform()->pointerAxisHorizontal(5.0, timestamp++);
     kwinApp()->platform()->keyboardKeyReleased(modifier, timestamp++);
-    QEXPECT_FAIL("", "Axis not yet handled", Continue);
     QCOMPARE(triggeredSpy.count(), 2);
 }
 
diff --git a/keyboard_input.cpp b/keyboard_input.cpp
index 3a6880a5d9..7781876888 100644
--- a/keyboard_input.cpp
+++ b/keyboard_input.cpp
@@ -95,6 +95,12 @@ Xkb::Xkb(InputRedirection *input)
         xkb_context_set_log_level(m_context, XKB_LOG_LEVEL_DEBUG);
         xkb_context_set_log_fn(m_context, &xkbLogHandler);
     }
+
+    auto resetModOnlyShortcut = [this] {
+        m_modOnlyShortcut.modifier = Qt::NoModifier;
+    };
+    QObject::connect(m_input, &InputRedirection::pointerButtonStateChanged, resetModOnlyShortcut);
+    QObject::connect(m_input, &InputRedirection::pointerAxisChanged, resetModOnlyShortcut);
 }
 
 Xkb::~Xkb()
@@ -243,7 +249,8 @@ void Xkb::updateKey(uint32_t key, InputRedirection::KeyboardKeyState state)
     updateModifiers();
     if (state == InputRedirection::KeyboardKeyPressed) {
         m_modOnlyShortcut.pressCount++;
-        if (m_modOnlyShortcut.pressCount == 1) {
+        if (m_modOnlyShortcut.pressCount == 1 &&
+            m_input->qtButtonStates() == Qt::NoButton) {
             m_modOnlyShortcut.modifier = Qt::KeyboardModifier(int(m_modifiers));
         } else {
             m_modOnlyShortcut.modifier = Qt::NoModifier;