diff --git a/src/plugins/overview/qml/main.qml b/src/plugins/overview/qml/main.qml
index 3e96601c93..89851eb13c 100644
--- a/src/plugins/overview/qml/main.qml
+++ b/src/plugins/overview/qml/main.qml
@@ -541,7 +541,7 @@ FocusScope {
                     }
                     onDropped: (drop) => {
                         drop.accepted = true;
-                        if (drag.source instanceof Item) {
+                        if (drop.keys.includes("kwin-desktop")) {
                             // dragging a desktop as a whole
                             if (drag.source === mainBackground) {
                                 drop.action = Qt.IgnoreAction;
@@ -607,6 +607,7 @@ FocusScope {
                     Drag.supportedActions: Qt.MoveAction
                     Drag.source: mainBackground
                     Drag.hotSpot: Qt.point(width * 0.5, height * 0.5)
+                    Drag.keys: ["kwin-desktop"]
 
                     layout.mode: effect.layout
                     focus: current
diff --git a/src/plugins/private/qml/WindowHeapDelegate.qml b/src/plugins/private/qml/WindowHeapDelegate.qml
index 6e5b7ba8df..bcf9bac79f 100644
--- a/src/plugins/private/qml/WindowHeapDelegate.qml
+++ b/src/plugins/private/qml/WindowHeapDelegate.qml
@@ -103,6 +103,7 @@ Item {
         Drag.hotSpot: Qt.point(
             thumb.activeDragHandler.centroid.pressPosition.x * thumb.targetScale,
             thumb.activeDragHandler.centroid.pressPosition.y * thumb.targetScale)
+        Drag.keys: ["kwin-window"]
 
         onXChanged: effect.checkItemDraggedOutOfScreen(thumbSource)
         onYChanged: effect.checkItemDraggedOutOfScreen(thumbSource)