diff --git a/autotests/integration/buttonrebind_test.cpp b/autotests/integration/buttonrebind_test.cpp
index 7dedb22437..a31dcf0770 100644
--- a/autotests/integration/buttonrebind_test.cpp
+++ b/autotests/integration/buttonrebind_test.cpp
@@ -89,6 +89,7 @@ void TestButtonRebind::testKey_data()
     QTest::newRow("delete") << QKeySequence(Qt::Key_Delete) << QList<quint32>{KEY_DELETE};
     QTest::newRow("keypad delete") << QKeySequence(Qt::KeypadModifier | Qt::Key_Delete) << QList<quint32>{KEY_KPDOT};
     QTest::newRow("keypad enter") << QKeySequence(Qt::KeypadModifier | Qt::Key_Enter) << QList<quint32>{KEY_KPENTER};
+    QTest::newRow("exclamation mark") << QKeySequence(Qt::Key_Exclam) << QList<quint32>{KEY_LEFTSHIFT, KEY_1};
 }
 
 void TestButtonRebind::testKey()
diff --git a/src/inputmethod.cpp b/src/inputmethod.cpp
index 3d9633b8f8..726ed997bd 100644
--- a/src/inputmethod.cpp
+++ b/src/inputmethod.cpp
@@ -71,7 +71,7 @@ static std::vector<quint32> textToKey(const QString &text)
     for (xkb_keysym_t sym : syms) {
         auto code = input()->keyboard()->xkb()->keycodeFromKeysym(sym);
         if (code) {
-            keyCode = code;
+            keyCode = code->first;
             break;
         }
     }
diff --git a/src/plugins/buttonrebinds/buttonrebindsfilter.cpp b/src/plugins/buttonrebinds/buttonrebindsfilter.cpp
index a0d36e6881..3ac876117e 100644
--- a/src/plugins/buttonrebinds/buttonrebindsfilter.cpp
+++ b/src/plugins/buttonrebinds/buttonrebindsfilter.cpp
@@ -362,10 +362,12 @@ bool ButtonRebindsFilter::sendKeySequence(const QKeySequence &keys, bool pressed
     }
     // KKeyServer returns upper case syms, lower it to not confuse modifiers handling
     std::optional<int> keyCode;
+    std::optional<int> level;
     for (int sym : syms) {
         auto code = KWin::input()->keyboard()->xkb()->keycodeFromKeysym(sym);
         if (code) {
-            keyCode = code;
+            keyCode = code->first;
+            level = code->second;
             break;
         }
     }
@@ -376,7 +378,7 @@ bool ButtonRebindsFilter::sendKeySequence(const QKeySequence &keys, bool pressed
 
     RebindScope scope;
 
-    if (key & Qt::ShiftModifier) {
+    if (key & Qt::ShiftModifier || level == 1) {
         sendKey(KEY_LEFTSHIFT);
     }
     if (key & Qt::ControlModifier) {
diff --git a/src/xkb.cpp b/src/xkb.cpp
index 2be28e5bae..545fa0b31f 100644
--- a/src/xkb.cpp
+++ b/src/xkb.cpp
@@ -1105,7 +1105,7 @@ void Xkb::setSeat(SeatInterface *seat)
     m_seat = QPointer<SeatInterface>(seat);
 }
 
-std::optional<int> Xkb::keycodeFromKeysym(xkb_keysym_t keysym)
+std::optional<std::pair<int, int>> Xkb::keycodeFromKeysym(xkb_keysym_t keysym)
 {
     auto layout = xkb_state_serialize_layout(m_state, XKB_STATE_LAYOUT_EFFECTIVE);
     const xkb_keycode_t max = xkb_keymap_max_keycode(m_keymap);
@@ -1116,7 +1116,7 @@ std::optional<int> Xkb::keycodeFromKeysym(xkb_keysym_t keysym)
             uint num_syms = xkb_keymap_key_get_syms_by_level(m_keymap, keycode, layout, currentLevel, &syms);
             for (uint sym = 0; sym < num_syms; sym++) {
                 if (syms[sym] == keysym) {
-                    return {keycode - EVDEV_OFFSET};
+                    return {{keycode - EVDEV_OFFSET, currentLevel}};
                 }
             }
         }
diff --git a/src/xkb.h b/src/xkb.h
index f8f3c3ef8d..9fe59486e9 100644
--- a/src/xkb.h
+++ b/src/xkb.h
@@ -116,7 +116,10 @@ public:
     void setSeat(SeatInterface *seat);
     QByteArray keymapContents() const;
 
-    std::optional<int> keycodeFromKeysym(xkb_keysym_t keysym);
+    /**
+     * Returns a pair of <keycode, level> for the given keysym.
+     */
+    std::optional<std::pair<int, int>> keycodeFromKeysym(xkb_keysym_t keysym);
 
     /**
      * Returns list of candidate keysyms corresponding to the given Qt key.