diff --git a/src/input.cpp b/src/input.cpp
index 52c5e4abd1..e26865ea8d 100644
--- a/src/input.cpp
+++ b/src/input.cpp
@@ -29,6 +29,7 @@
 #include "pointer_input.h"
 #include "tablet_input.h"
 #include "touch_input.h"
+#include "wayland/abstract_data_source.h"
 #include "wayland/xdgtopleveldrag_v1.h"
 #if KWIN_BUILD_X11
 #include "x11window.h"
@@ -2386,6 +2387,16 @@ class DragAndDropInputFilter : public QObject, public InputEventFilter
 public:
     DragAndDropInputFilter()
     {
+        connect(waylandServer()->seat(), &SeatInterface::dragStarted, this, []() {
+            AbstractDataSource *dragSource = waylandServer()->seat()->dragSource();
+            Q_ASSERT(dragSource);
+            dragSource->setKeyboardModifiers(input()->keyboardModifiers());
+
+            connect(input(), &InputRedirection::keyboardModifiersChanged, dragSource, [dragSource](Qt::KeyboardModifiers mods) {
+                dragSource->setKeyboardModifiers(mods);
+            });
+        });
+
         m_raiseTimer.setSingleShot(true);
         m_raiseTimer.setInterval(1000);
         connect(&m_raiseTimer, &QTimer::timeout, this, &DragAndDropInputFilter::raiseDragTarget);
diff --git a/src/keyboard_input.cpp b/src/keyboard_input.cpp
index 97befc3e9f..55b4cdbb32 100644
--- a/src/keyboard_input.cpp
+++ b/src/keyboard_input.cpp
@@ -103,7 +103,11 @@ public:
         if (event->isAutoRepeat()) {
             return;
         }
-        const Qt::KeyboardModifiers mods = event->modifiers();
+
+        // QKeyEvent::modifiers differs from the superclass QInputEvent::modifiers
+        // QKeyEvent tries to special case an old QtXCB behaviour and assumes modifiers aren't processed at
+        // the time of the release and inverts the logic. This is not the case for kwin
+        const Qt::KeyboardModifiers mods = event->QInputEvent::modifiers();
         if (mods == m_modifiers) {
             return;
         }
diff --git a/src/wayland/abstract_data_source.cpp b/src/wayland/abstract_data_source.cpp
index 43ec21e533..8d882d2f78 100644
--- a/src/wayland/abstract_data_source.cpp
+++ b/src/wayland/abstract_data_source.cpp
@@ -14,6 +14,19 @@ AbstractDataSource::AbstractDataSource(QObject *parent)
 {
 }
 
+void AbstractDataSource::setKeyboardModifiers(Qt::KeyboardModifiers heldModifiers)
+{
+    if (m_heldModifiers == heldModifiers) {
+        return;
+    }
+    m_heldModifiers = heldModifiers;
+    Q_EMIT keyboardModifiersChanged();
+}
+
+Qt::KeyboardModifiers AbstractDataSource::keyboardModifiers() const
+{
+    return m_heldModifiers;
+}
 } // namespace KWin
 
 #include "moc_abstract_data_source.cpp"
diff --git a/src/wayland/abstract_data_source.h b/src/wayland/abstract_data_source.h
index af5f9005bd..1962c44f2d 100644
--- a/src/wayland/abstract_data_source.h
+++ b/src/wayland/abstract_data_source.h
@@ -97,16 +97,25 @@ public:
         return nullptr;
     };
 
+    /**
+     * Called from kwin core code, this updates the keyboard modifiers currently pressed
+     * which can be used to determine the best DND action
+     */
+    void setKeyboardModifiers(Qt::KeyboardModifiers heldModifiers);
+    Qt::KeyboardModifiers keyboardModifiers() const;
+
 Q_SIGNALS:
     void aboutToBeDestroyed();
 
     void mimeTypeOffered(const QString &);
     void supportedDragAndDropActionsChanged();
+    void keyboardModifiersChanged();
 
 protected:
     explicit AbstractDataSource(QObject *parent = nullptr);
 
 private:
+    Qt::KeyboardModifiers m_heldModifiers;
     bool m_dndCancelled = false;
     bool m_dndDropped = false;
 };
diff --git a/src/wayland/datadevice.cpp b/src/wayland/datadevice.cpp
index 0bff2894eb..0edff59e09 100644
--- a/src/wayland/datadevice.cpp
+++ b/src/wayland/datadevice.cpp
@@ -214,17 +214,33 @@ void DataDeviceInterface::drop()
         d->drag.sourceActionConnection = QMetaObject::Connection();
         disconnect(d->drag.targetActionConnection);
         d->drag.targetActionConnection = QMetaObject::Connection();
+        disconnect(d->drag.keyboardModifiersConnection);
+        d->drag.keyboardModifiersConnection = QMetaObject::Connection();
     }
 }
 
-static DataDeviceManagerInterface::DnDAction chooseDndAction(AbstractDataSource *source, DataOfferInterface *offer)
+static DataDeviceManagerInterface::DnDAction chooseDndAction(AbstractDataSource *source, DataOfferInterface *offer, Qt::KeyboardModifiers keyboardModifiers)
 {
+    // first compositor picks an action if modifiers are pressed and it's supported both sides
+    if (keyboardModifiers.testFlag(Qt::ControlModifier)) {
+        if (source->supportedDragAndDropActions().testFlag(DataDeviceManagerInterface::DnDAction::Copy) && offer->supportedDragAndDropActions().has_value() && offer->supportedDragAndDropActions()->testFlag(DataDeviceManagerInterface::DnDAction::Copy)) {
+            return DataDeviceManagerInterface::DnDAction::Copy;
+        }
+    }
+    if (keyboardModifiers.testFlag(Qt::ShiftModifier)) {
+        if (source->supportedDragAndDropActions().testFlag(DataDeviceManagerInterface::DnDAction::Move) && offer->supportedDragAndDropActions().has_value() && offer->supportedDragAndDropActions()->testFlag(DataDeviceManagerInterface::DnDAction::Move)) {
+            return DataDeviceManagerInterface::DnDAction::Move;
+        }
+    }
+
+    // otherwise we pick the preferred action from the target if the source supported it
     if (offer->preferredDragAndDropAction().has_value()) {
         if (source->supportedDragAndDropActions().testFlag(*offer->preferredDragAndDropAction())) {
             return *offer->preferredDragAndDropAction();
         }
     }
 
+    // finally pick something everyone supports in a deterministic fashion
     if (offer->supportedDragAndDropActions().has_value()) {
         for (const auto &action : {DataDeviceManagerInterface::DnDAction::Copy, DataDeviceManagerInterface::DnDAction::Move, DataDeviceManagerInterface::DnDAction::Ask}) {
             if (source->supportedDragAndDropActions().testFlag(action) && offer->supportedDragAndDropActions()->testFlag(action)) {
@@ -260,6 +276,10 @@ void DataDeviceInterface::updateDragTarget(SurfaceInterface *surface, quint32 se
             disconnect(d->drag.targetActionConnection);
             d->drag.targetActionConnection = QMetaObject::Connection();
         }
+        if (d->drag.keyboardModifiersConnection) {
+            disconnect(d->drag.keyboardModifiersConnection);
+            d->drag.keyboardModifiersConnection = QMetaObject::Connection();
+        }
         // don't update serial, we need it
     }
     auto dragSource = d->seat->dragSource();
@@ -323,13 +343,20 @@ void DataDeviceInterface::updateDragTarget(SurfaceInterface *surface, quint32 se
     }
     d->send_enter(serial, surface->resource(), wl_fixed_from_double(pos.x()), wl_fixed_from_double(pos.y()), offer ? offer->resource() : nullptr);
     if (offer) {
-        auto matchOffers = [dragSource, offer] {
-            const DataDeviceManagerInterface::DnDAction action = chooseDndAction(dragSource, offer);
+        auto matchOffers = [this, dragSource, offer] {
+            Qt::KeyboardModifiers keyboardModifiers;
+            if (d->seat->isDrag()) { // ignore keyboard modifiers when in "ask" negotiation
+                keyboardModifiers = dragSource->keyboardModifiers();
+            }
+
+            const DataDeviceManagerInterface::DnDAction action = chooseDndAction(dragSource, offer, keyboardModifiers);
             offer->dndAction(action);
             dragSource->dndAction(action);
         };
+        matchOffers();
         d->drag.targetActionConnection = connect(offer, &DataOfferInterface::dragAndDropActionsChanged, dragSource, matchOffers);
         d->drag.sourceActionConnection = connect(dragSource, &AbstractDataSource::supportedDragAndDropActionsChanged, offer, matchOffers);
+        d->drag.keyboardModifiersConnection = connect(dragSource, &AbstractDataSource::keyboardModifiersChanged, offer, matchOffers);
     }
 }
 
diff --git a/src/wayland/datadevice_p.h b/src/wayland/datadevice_p.h
index dbf60b5b8c..2205586e58 100644
--- a/src/wayland/datadevice_p.h
+++ b/src/wayland/datadevice_p.h
@@ -41,6 +41,7 @@ public:
         QMetaObject::Connection posConnection;
         QMetaObject::Connection sourceActionConnection;
         QMetaObject::Connection targetActionConnection;
+        QMetaObject::Connection keyboardModifiersConnection;
         quint32 serial = 0;
     };
     Drag drag;