diff --git a/src/main_wayland.cpp b/src/main_wayland.cpp
index 1574a77780..37a3a7e149 100644
--- a/src/main_wayland.cpp
+++ b/src/main_wayland.cpp
@@ -202,7 +202,7 @@ void ApplicationWayland::refreshSettings(const KConfigGroup &group, const QByteA
     }
 
     if (group.name() == "Wayland" && names.contains("EnablePrimarySelection")) {
-        waylandServer()->seat()->setPrimarySelectionEnabled(group.readEntry("EnablePrimarySelection", true));
+        waylandServer()->setEnablePrimarySelection(group.readEntry("EnablePrimarySelection", true));
     }
 }
 
diff --git a/src/wayland/autotests/server/test_datacontrol_interface.cpp b/src/wayland/autotests/server/test_datacontrol_interface.cpp
index c6d352b1ce..b86ac5757c 100644
--- a/src/wayland/autotests/server/test_datacontrol_interface.cpp
+++ b/src/wayland/autotests/server/test_datacontrol_interface.cpp
@@ -120,16 +120,11 @@ public:
     void requestData(const QString &mimeType, qint32 fd) override
     {
     };
-    void cancel() override
-    {
-        Q_EMIT cancelled();
-    };
+    void cancel() override{};
     QStringList mimeTypes() const override
     {
         return {"text/test1", "text/test2"};
     }
-Q_SIGNALS:
-    void cancelled();
 };
 
 // The test itself
@@ -146,7 +141,6 @@ private Q_SLOTS:
     void testCopyFromControl();
     void testCopyFromControlPrimarySelection();
     void testKlipperCase();
-    void testPrimarySelectionDisabled();
 
 private:
     KWayland::Client::ConnectionThread *m_connection;
@@ -302,32 +296,6 @@ void DataControlInterfaceTest::testCopyToControlPrimarySelection()
     QCOMPARE(offer->receivedOffers()[1], "text/test2");
 }
 
-void DataControlInterfaceTest::testPrimarySelectionDisabled()
-{
-    // we set a dummy data source on the seat using abstract client directly
-    // then confirm we receive the offer despite not having a surface
-
-    // disable primary selection
-    m_seat->setPrimarySelectionEnabled(false);
-
-    std::unique_ptr<DataControlDevice> dataControlDevice(new DataControlDevice);
-    dataControlDevice->init(m_dataControlDeviceManager->get_data_device(*m_clientSeat));
-
-    QSignalSpy newOfferSpy(dataControlDevice.get(), &DataControlDevice::dataControlOffer);
-    QSignalSpy selectionSpy(dataControlDevice.get(), &DataControlDevice::primary_selection);
-
-    std::unique_ptr<TestDataSource> testSelection(new TestDataSource);
-    QSignalSpy cancelSpy(testSelection.get(), &TestDataSource::cancelled);
-
-    m_seat->setPrimarySelection(testSelection.get());
-
-    // selection will be sent after we've been sent a new offer object and the mimes have been sent to that object
-    cancelSpy.wait();
-
-    QCOMPARE(newOfferSpy.count(), 0);
-    QCOMPARE(cancelSpy.count(), 1);
-}
-
 void DataControlInterfaceTest::testCopyFromControl()
 {
     // we create a data device and set a selection
diff --git a/src/wayland/seat_interface.cpp b/src/wayland/seat_interface.cpp
index af33046cb3..de95b7280b 100644
--- a/src/wayland/seat_interface.cpp
+++ b/src/wayland/seat_interface.cpp
@@ -1291,14 +1291,6 @@ AbstractDataSource *SeatInterface::primarySelection() const
     return d->currentPrimarySelection;
 }
 
-void SeatInterface::setPrimarySelectionEnabled(bool enabled)
-{
-    if (enabled != d->primarySelectionEnabled) {
-        setPrimarySelection(nullptr);
-        d->primarySelectionEnabled = enabled;
-    }
-}
-
 void SeatInterface::setPrimarySelection(AbstractDataSource *selection)
 {
     if (d->currentPrimarySelection == selection) {
@@ -1309,13 +1301,6 @@ void SeatInterface::setPrimarySelection(AbstractDataSource *selection)
         disconnect(d->currentPrimarySelection, nullptr, this, nullptr);
     }
 
-    if (!d->primarySelectionEnabled) {
-        if (selection) {
-            selection->cancel();
-        }
-        return;
-    }
-
     if (selection) {
         auto cleanup = [this]() {
             setPrimarySelection(nullptr);
diff --git a/src/wayland/seat_interface.h b/src/wayland/seat_interface.h
index d99d71fd87..d79494d380 100644
--- a/src/wayland/seat_interface.h
+++ b/src/wayland/seat_interface.h
@@ -170,8 +170,6 @@ public:
     void setTimestamp(quint32 time);
     quint32 timestamp() const;
 
-    void setPrimarySelectionEnabled(bool enabled);
-
     /**
      * @name Drag'n'Drop related methods
      */
diff --git a/src/wayland/seat_interface_p.h b/src/wayland/seat_interface_p.h
index de2712d09a..ff9457657c 100644
--- a/src/wayland/seat_interface_p.h
+++ b/src/wayland/seat_interface_p.h
@@ -67,7 +67,6 @@ public:
     // the last thing copied into the clipboard content
     AbstractDataSource *currentSelection = nullptr;
     AbstractDataSource *currentPrimarySelection = nullptr;
-    bool primarySelectionEnabled = true;
 
     // Pointer related members
     struct Pointer
diff --git a/src/wayland_server.cpp b/src/wayland_server.cpp
index d1bbe5aa72..eaa80a9450 100644
--- a/src/wayland_server.cpp
+++ b/src/wayland_server.cpp
@@ -320,6 +320,16 @@ void WaylandServer::handleOutputDisabled(Output *output)
     }
 }
 
+void WaylandServer::setEnablePrimarySelection(bool enable)
+{
+    if (!enable && m_primarySelectionDeviceManager != nullptr) {
+        delete m_primarySelectionDeviceManager;
+        m_primarySelectionDeviceManager = nullptr;
+    } else if (enable && m_primarySelectionDeviceManager == nullptr) {
+        m_primarySelectionDeviceManager = new PrimarySelectionDeviceManagerV1Interface(m_display, m_display);
+    }
+}
+
 bool WaylandServer::start()
 {
     return m_display->start();
@@ -420,8 +430,7 @@ bool WaylandServer::init(InitializationFlags flags)
     new DataControlDeviceManagerV1Interface(m_display, m_display);
 
     const auto kwinConfig = kwinApp()->config();
-    m_seat->setPrimarySelectionEnabled(kwinConfig->group("Wayland").readEntry("EnablePrimarySelection", true));
-    new PrimarySelectionDeviceManagerV1Interface(m_display, m_display);
+    setEnablePrimarySelection(kwinConfig->group("Wayland").readEntry("EnablePrimarySelection", true));
 
     m_idle = new IdleInterface(m_display, m_display);
     auto idleInhibition = new IdleInhibition(m_idle);
diff --git a/src/wayland_server.h b/src/wayland_server.h
index b0bc11c507..45caa8c1e3 100644
--- a/src/wayland_server.h
+++ b/src/wayland_server.h
@@ -47,6 +47,7 @@ class LinuxDmaBufV1ClientBuffer;
 class TabletManagerV2Interface;
 class KeyboardShortcutsInhibitManagerV1Interface;
 class XdgDecorationManagerV1Interface;
+class PrimarySelectionDeviceManagerV1Interface;
 class XWaylandKeyboardGrabManagerV1Interface;
 class ContentTypeManagerV1Interface;
 class DrmLeaseManagerV1;
@@ -227,6 +228,7 @@ public:
     {
         m_linuxDmabufBuffers.remove(buffer);
     }
+    void setEnablePrimarySelection(bool enable);
 
     /**
      * Returns the first socket name that can be used to connect to this server.
@@ -293,6 +295,7 @@ private:
     KWaylandServer::ClientConnection *m_screenLockerClientConnection = nullptr;
     KWaylandServer::XdgForeignV2Interface *m_XdgForeign = nullptr;
     XdgActivationV1Integration *m_xdgActivationIntegration = nullptr;
+    KWaylandServer::PrimarySelectionDeviceManagerV1Interface *m_primarySelectionDeviceManager = nullptr;
     KWaylandServer::XWaylandKeyboardGrabManagerV1Interface *m_xWaylandKeyboardGrabManager = nullptr;
     KWaylandServer::ContentTypeManagerV1Interface *m_contentTypeManager = nullptr;
     KWaylandServer::TearingControlManagerV1Interface *m_tearingControlInterface = nullptr;