diff --git a/cursor.cpp b/cursor.cpp
index 8fe4c767c3..031de0b3fe 100644
--- a/cursor.cpp
+++ b/cursor.cpp
@@ -430,6 +430,13 @@ InputRedirectionCursor::InputRedirectionCursor(QObject *parent)
 #ifndef KCMRULES
     connect(input(), &InputRedirection::keyboardModifiersChanged,
             this, &InputRedirectionCursor::slotModifiersChanged);
+    connect(kwinApp(), &Application::x11ConnectionChanged, this,
+        [this] {
+            if (isCursorTracking()) {
+                doStartCursorTracking();
+            }
+        }, Qt::QueuedConnection
+    );
 #endif
 }
 
@@ -464,12 +471,18 @@ void InputRedirectionCursor::slotPointerButtonChanged()
 
 void InputRedirectionCursor::doStartCursorTracking()
 {
+    if (!kwinApp()->x11Connection()) {
+        return;
+    }
     xcb_xfixes_select_cursor_input(connection(), rootWindow(), XCB_XFIXES_CURSOR_NOTIFY_MASK_DISPLAY_CURSOR);
     // TODO: also track the Wayland cursor
 }
 
 void InputRedirectionCursor::doStopCursorTracking()
 {
+    if (!kwinApp()->x11Connection()) {
+        return;
+    }
     xcb_xfixes_select_cursor_input(connection(), rootWindow(), 0);
     // TODO: also track the Wayland cursor
 }
diff --git a/cursor.h b/cursor.h
index 5e2342fb2a..df224e4c2e 100644
--- a/cursor.h
+++ b/cursor.h
@@ -186,6 +186,7 @@ protected:
      * to overwrite to disable platform specific code for the tracking.
      */
     virtual void doStopCursorTracking();
+    bool isCursorTracking() const;
     /**
      * Provides the actual internal cursor position to inheriting classes. If an inheriting class needs
      * access to the cursor position this method should be used instead of the static @link pos, as
@@ -295,6 +296,11 @@ inline int Cursor::themeSize() const
     return m_themeSize;
 }
 
+inline bool Cursor::isCursorTracking() const
+{
+    return m_cursorTrackingCounter > 0;
+}
+
 }
 
 #endif // KWIN_CURSOR_H