diff --git a/src/xwayland/dnd.cpp b/src/xwayland/dnd.cpp
index a2b5477be1..ef62f82599 100644
--- a/src/xwayland/dnd.cpp
+++ b/src/xwayland/dnd.cpp
@@ -106,11 +106,10 @@ void Dnd::doHandleXfixesNotify(xcb_xfixes_selection_notify_event_t *event)
         return;
     }
     createX11Source(event);
-    X11Source *source = x11Source();
-    if (!source) {
-        return;
+    if (X11Source *source = x11Source()) {
+        SeatInterface *seat = waylandServer()->seat();
+        seat->startDrag(source->dataSource(), seat->focusedPointerSurface(), seat->pointerButtonSerial(Qt::LeftButton));
     }
-    m_currentDrag = new XToWlDrag(source, this);
 }
 
 void Dnd::x11OfferLost()
@@ -142,25 +141,24 @@ DragEventReply Dnd::dragMoveFilter(Window *target)
 
 void Dnd::startDrag()
 {
-    auto dragSource = waylandServer()->seat()->dragSource();
-    if (qobject_cast<XwlDataSource *>(dragSource)) {
-        return;
-    }
-
     // There can only ever be one Wl native drag at the same time.
     Q_ASSERT(!m_currentDrag);
 
-    // New Wl to X drag, init drag and Wl source.
-    m_currentDrag = new WlToXDrag(this);
-    auto source = new WlSource(this);
-    source->setDataSourceIface(dragSource);
-    connect(dragSource, &AbstractDataSource::aboutToBeDestroyed, this, [this, source] {
-        if (source == wlSource()) {
-            setWlSource(nullptr);
-        }
-    });
-    setWlSource(source);
-    ownSelection(true);
+    auto dragSource = waylandServer()->seat()->dragSource();
+    if (qobject_cast<XwlDataSource *>(dragSource)) {
+        m_currentDrag = new XToWlDrag(x11Source(), this);
+    } else {
+        m_currentDrag = new WlToXDrag(this);
+        auto source = new WlSource(this);
+        source->setDataSourceIface(dragSource);
+        connect(dragSource, &AbstractDataSource::aboutToBeDestroyed, this, [this, source] {
+            if (source == wlSource()) {
+                setWlSource(nullptr);
+            }
+        });
+        setWlSource(source);
+        ownSelection(true);
+    }
 }
 
 void Dnd::endDrag()
diff --git a/src/xwayland/drag_x.cpp b/src/xwayland/drag_x.cpp
index 27fad37167..93d0c9d1f6 100644
--- a/src/xwayland/drag_x.cpp
+++ b/src/xwayland/drag_x.cpp
@@ -76,7 +76,7 @@ XToWlDrag::XToWlDrag(X11Source *source, Dnd *dnd)
     connect(source, &X11Source::transferReady, this, [this](xcb_atom_t target, qint32 fd) {
         m_dataRequests << QPair<xcb_timestamp_t, bool>(m_source->timestamp(), false);
     });
-    connect(&m_selectionSource, &XwlDataSource::dropped, this, [this] {
+    connect(source->dataSource(), &XwlDataSource::dropped, this, [this] {
         m_performed = true;
         if (m_visit) {
             connect(m_visit, &WlVisit::finish, this, [this](WlVisit *visit) {
@@ -97,21 +97,16 @@ XToWlDrag::XToWlDrag(X11Source *source, Dnd *dnd)
         // Dave do we need this async finish check anymore?
         checkForFinished();
     });
-    connect(&m_selectionSource, &XwlDataSource::finished, this, [this] {
+    connect(source->dataSource(), &XwlDataSource::finished, this, [this] {
         checkForFinished();
     });
-    connect(&m_selectionSource, &XwlDataSource::cancelled, this, [this] {
+    connect(source->dataSource(), &XwlDataSource::cancelled, this, [this] {
         if (m_visit && !m_visit->leave()) {
             connect(m_visit, &WlVisit::finish, this, &XToWlDrag::checkForFinished);
         }
         checkForFinished();
     });
-    connect(&m_selectionSource, &XwlDataSource::dataRequested, source, &X11Source::startTransfer);
-
-    auto *seat = waylandServer()->seat();
-    int serial = waylandServer()->seat()->pointerButtonSerial(Qt::LeftButton);
-    // we know we are the focussed surface as dnd checks
-    seat->startDrag(&m_selectionSource, seat->focusedPointerSurface(), serial);
+    connect(source->dataSource(), &XwlDataSource::dataRequested, source, &X11Source::startTransfer);
 }
 
 XToWlDrag::~XToWlDrag()
@@ -172,12 +167,12 @@ bool XToWlDrag::handleClientMessage(xcb_client_message_event_t *event)
 
 void XToWlDrag::setDragAndDropAction(DnDAction action)
 {
-    m_selectionSource.setSupportedDndActions(action);
+    m_source->dataSource()->setSupportedDndActions(action);
 }
 
 DnDAction XToWlDrag::selectedDragAndDropAction()
 {
-    return m_selectionSource.selectedDndAction();
+    return m_source->dataSource()->selectedDndAction();
 }
 
 void XToWlDrag::setOffers(const Mimes &offers)
@@ -202,7 +197,7 @@ void XToWlDrag::setOffers(const Mimes &offers)
     for (const auto &mimePair : offers) {
         mimeTypes.append(mimePair.first);
     }
-    m_selectionSource.setMimeTypes(mimeTypes);
+    m_source->dataSource()->setMimeTypes(mimeTypes);
     setDragTarget();
 }
 
@@ -235,7 +230,7 @@ bool XToWlDrag::checkForFinished()
     if (!m_visit->finished()) {
         return false;
     }
-    if (m_dataRequests.size() == 0 && m_selectionSource.isAccepted()) {
+    if (m_dataRequests.size() == 0 && m_source->dataSource()->isAccepted()) {
         // need to wait for first data request
         return false;
     }
diff --git a/src/xwayland/drag_x.h b/src/xwayland/drag_x.h
index 65bfb8f33e..be7f7857c6 100644
--- a/src/xwayland/drag_x.h
+++ b/src/xwayland/drag_x.h
@@ -58,8 +58,6 @@ private:
     Dnd *const m_dnd;
     Mimes m_offers;
 
-    XwlDataSource m_selectionSource;
-
     X11Source *m_source;
     QList<QPair<xcb_timestamp_t, bool>> m_dataRequests;
 
diff --git a/src/xwayland/selection_source.cpp b/src/xwayland/selection_source.cpp
index 5d81c272a4..d5818a941d 100644
--- a/src/xwayland/selection_source.cpp
+++ b/src/xwayland/selection_source.cpp
@@ -7,6 +7,7 @@
     SPDX-License-Identifier: GPL-2.0-or-later
 */
 #include "selection_source.h"
+#include "datasource.h"
 #include "selection.h"
 #include "transfer.h"
 
@@ -161,10 +162,15 @@ bool WlSource::checkStartTransfer(xcb_selection_request_event_t *event)
 
 X11Source::X11Source(Selection *selection, xcb_xfixes_selection_notify_event_t *event)
     : SelectionSource(selection)
+    , m_dataSource(std::make_unique<XwlDataSource>())
 {
     setTimestamp(event->timestamp);
 }
 
+X11Source::~X11Source()
+{
+}
+
 void X11Source::getTargets()
 {
     xcb_connection_t *xcbConn = kwinApp()->x11Connection();
diff --git a/src/xwayland/selection_source.h b/src/xwayland/selection_source.h
index 2910e304d6..ac5f52882f 100644
--- a/src/xwayland/selection_source.h
+++ b/src/xwayland/selection_source.h
@@ -27,6 +27,7 @@ class AbstractDataSource;
 namespace Xwl
 {
 class Selection;
+class XwlDataSource;
 
 /**
  * Base class representing a data source.
@@ -112,6 +113,7 @@ class X11Source : public SelectionSource
 
 public:
     X11Source(Selection *selection, xcb_xfixes_selection_notify_event_t *event);
+    ~X11Source() override;
 
     void getTargets();
 
@@ -121,6 +123,11 @@ public:
     }
     void setOffers(const Mimes &offers);
 
+    XwlDataSource *dataSource() const
+    {
+        return m_dataSource.get();
+    }
+
     bool handleSelectionNotify(xcb_selection_notify_event_t *event);
 
     void setRequestor(xcb_window_t window)
@@ -137,6 +144,7 @@ private:
     void handleTargets();
 
     Mimes m_offers;
+    std::unique_ptr<XwlDataSource> m_dataSource;
 
     Q_DISABLE_COPY(X11Source)
 };