diff --git a/src/xwl/datasource.h b/src/xwl/datasource.h
index 58c2977265..ac45f09830 100644
--- a/src/xwl/datasource.h
+++ b/src/xwl/datasource.h
@@ -47,12 +47,17 @@ public:
     void dndFinished() override {
         Q_EMIT finished();
     }
+    void dndCancelled() override {
+        Q_EMIT cancelled();
+    }
+
     bool isAccepted() const override;
 
 Q_SIGNALS:
     void dataRequested(const QString &mimeType, qint32 fd);
     void dropped();
     void finished();
+    void cancelled();
 
 private:
     QStringList m_mimeTypes;
diff --git a/src/xwl/drag_x.cpp b/src/xwl/drag_x.cpp
index 6811fd6bdd..fe0acb07ee 100644
--- a/src/xwl/drag_x.cpp
+++ b/src/xwl/drag_x.cpp
@@ -101,6 +101,12 @@ XToWlDrag::XToWlDrag(X11Source *source)
     connect(&m_selectionSource, &XwlDataSource::finished, this, [this] {
         checkForFinished();
     });
+    connect(&m_selectionSource, &XwlDataSource::cancelled, this, [this] {
+        if (m_visit && !m_visit->leave())  {
+            connect(m_visit, &WlVisit::finish, this, &XToWlDrag::checkForFinished);
+        }
+        checkForFinished();
+    });
     connect(&m_selectionSource, &XwlDataSource::dataRequested, source, &X11Source::startTransfer);
 
     auto *seat = waylandServer()->seat();
@@ -229,7 +235,7 @@ bool XToWlDrag::checkForFinished()
     if (!m_visit->finished()) {
         return false;
     }
-    if (m_dataRequests.size() == 0) {
+    if (m_dataRequests.size() == 0 && m_selectionSource.isAccepted()) {
         // need to wait for first data request
         return false;
     }