diff --git a/src/wayland/CMakeLists.txt b/src/wayland/CMakeLists.txt
index d466149f7c..d02137e0ce 100644
--- a/src/wayland/CMakeLists.txt
+++ b/src/wayland/CMakeLists.txt
@@ -215,7 +215,6 @@ target_sources(kwin PRIVATE
     appmenu_interface.cpp
     blur_interface.cpp
     clientbuffer.cpp
-    clientbufferintegration.cpp
     clientconnection.cpp
     compositor_interface.cpp
     contenttype_v1_interface.cpp
diff --git a/src/wayland/clientbuffer.cpp b/src/wayland/clientbuffer.cpp
index 5488251b2d..62fc619723 100644
--- a/src/wayland/clientbuffer.cpp
+++ b/src/wayland/clientbuffer.cpp
@@ -4,44 +4,22 @@
     SPDX-License-Identifier: LGPL-2.1-only OR LGPL-3.0-only OR LicenseRef-KDE-Accepted-LGPL
 */
 
-#include "clientbuffer.h"
-#include "clientbuffer_p.h"
-
-#include <wayland-server-protocol.h>
+#include "wayland/clientbuffer.h"
+#include "wayland/linuxdmabufv1clientbuffer.h"
+#include "wayland/shmclientbuffer.h"
 
 namespace KWaylandServer
 {
-ClientBuffer::ClientBuffer(std::unique_ptr<ClientBufferPrivate> &&d)
-    : d_ptr(std::move(d))
-{
-}
 
-ClientBuffer::ClientBuffer(wl_resource *resource, std::unique_ptr<ClientBufferPrivate> &&d)
-    : d_ptr(std::move(d))
+ClientBuffer *ClientBuffer::get(wl_resource *resource)
 {
-    initialize(resource);
-}
-
-ClientBuffer::~ClientBuffer()
-{
-}
-
-void ClientBuffer::initialize(wl_resource *resource)
-{
-    Q_D(ClientBuffer);
-    d->resource = resource;
-    connect(this, &GraphicsBuffer::dropped, [d]() {
-        d->resource = nullptr;
-    });
-    connect(this, &GraphicsBuffer::released, [d]() {
-        wl_buffer_send_release(d->resource);
-    });
-}
-
-wl_resource *ClientBuffer::resource() const
-{
-    Q_D(const ClientBuffer);
-    return d->resource;
+    if (auto buffer = LinuxDmaBufV1ClientBuffer::get(resource)) {
+        return buffer;
+    }
+    if (auto buffer = ShmClientBuffer::get(resource)) {
+        return buffer;
+    }
+    return nullptr;
 }
 
 } // namespace KWaylandServer
diff --git a/src/wayland/clientbuffer.h b/src/wayland/clientbuffer.h
index d0d7b86e25..ac8bc1fa4b 100644
--- a/src/wayland/clientbuffer.h
+++ b/src/wayland/clientbuffer.h
@@ -8,16 +8,10 @@
 
 #include "core/graphicsbuffer.h"
 
-#include <QImage>
-#include <QObject>
-#include <QSize>
-#include <memory>
-
 struct wl_resource;
 
 namespace KWaylandServer
 {
-class ClientBufferPrivate;
 
 /**
  * The ClientBuffer class represents a client buffer.
@@ -30,23 +24,9 @@ class ClientBufferPrivate;
 class KWIN_EXPORT ClientBuffer : public KWin::GraphicsBuffer
 {
     Q_OBJECT
-    Q_DECLARE_PRIVATE(ClientBuffer)
 
 public:
-    ~ClientBuffer() override;
-
-    /**
-     * Returns the wl_resource for this ClientBuffer. If the buffer is destroyed, @c null
-     * will be returned.
-     */
-    wl_resource *resource() const;
-
-protected:
-    ClientBuffer(std::unique_ptr<ClientBufferPrivate> &&d);
-    ClientBuffer(wl_resource *resource, std::unique_ptr<ClientBufferPrivate> &&d);
-
-    void initialize(wl_resource *resource);
-    std::unique_ptr<ClientBufferPrivate> d_ptr;
+    static ClientBuffer *get(wl_resource *resource);
 };
 
 } // namespace KWaylandServer
diff --git a/src/wayland/clientbuffer_p.h b/src/wayland/clientbuffer_p.h
deleted file mode 100644
index 3045f423bd..0000000000
--- a/src/wayland/clientbuffer_p.h
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
-    SPDX-FileCopyrightText: 2021 Vlad Zahorodnii <vlad.zahorodnii@kde.org>
-
-    SPDX-License-Identifier: LGPL-2.1-only OR LGPL-3.0-only OR LicenseRef-KDE-Accepted-LGPL
-*/
-
-#pragma once
-
-#include "clientbuffer.h"
-
-namespace KWaylandServer
-{
-class ClientBufferPrivate
-{
-public:
-    virtual ~ClientBufferPrivate()
-    {
-    }
-
-    wl_resource *resource = nullptr;
-};
-
-} // namespace KWaylandServer
diff --git a/src/wayland/clientbufferintegration.cpp b/src/wayland/clientbufferintegration.cpp
deleted file mode 100644
index 888301a1e6..0000000000
--- a/src/wayland/clientbufferintegration.cpp
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
-    SPDX-FileCopyrightText: 2021 Vlad Zahorodnii <vlad.zahorodnii@kde.org>
-
-    SPDX-License-Identifier: LGPL-2.1-only OR LGPL-3.0-only OR LicenseRef-KDE-Accepted-LGPL
-*/
-
-#include "clientbufferintegration.h"
-#include "display.h"
-#include "display_p.h"
-
-namespace KWaylandServer
-{
-ClientBufferIntegration::ClientBufferIntegration(Display *display)
-    : QObject(display)
-    , m_display(display)
-{
-    DisplayPrivate *displayPrivate = DisplayPrivate::get(display);
-    displayPrivate->bufferIntegrations.append(this);
-}
-
-ClientBufferIntegration::~ClientBufferIntegration()
-{
-    if (m_display) {
-        DisplayPrivate *displayPrivate = DisplayPrivate::get(m_display);
-        displayPrivate->bufferIntegrations.removeOne(this);
-    }
-}
-
-Display *ClientBufferIntegration::display() const
-{
-    return m_display;
-}
-
-ClientBuffer *ClientBufferIntegration::createBuffer(wl_resource *resource)
-{
-    return nullptr;
-}
-
-} // namespace KWaylandServer
diff --git a/src/wayland/clientbufferintegration.h b/src/wayland/clientbufferintegration.h
deleted file mode 100644
index 0c2fd6ee02..0000000000
--- a/src/wayland/clientbufferintegration.h
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
-    SPDX-FileCopyrightText: 2021 Vlad Zahorodnii <vlad.zahorodnii@kde.org>
-
-    SPDX-License-Identifier: LGPL-2.1-only OR LGPL-3.0-only OR LicenseRef-KDE-Accepted-LGPL
-*/
-
-#pragma once
-
-#include "clientbuffer.h"
-
-#include <QPointer>
-
-namespace KWaylandServer
-{
-class Display;
-
-class KWIN_EXPORT ClientBufferIntegration : public QObject
-{
-    Q_OBJECT
-
-public:
-    explicit ClientBufferIntegration(Display *display);
-    ~ClientBufferIntegration() override;
-
-    Display *display() const;
-
-    virtual ClientBuffer *createBuffer(wl_resource *resource);
-
-private:
-    QPointer<Display> m_display;
-};
-
-} // namespace KWaylandServer
diff --git a/src/wayland/display.cpp b/src/wayland/display.cpp
index cba2216705..f0c797d805 100644
--- a/src/wayland/display.cpp
+++ b/src/wayland/display.cpp
@@ -5,7 +5,6 @@
     SPDX-License-Identifier: LGPL-2.1-only OR LGPL-3.0-only OR LicenseRef-KDE-Accepted-LGPL
 */
 #include "display.h"
-#include "clientbufferintegration.h"
 #include "display_p.h"
 #include "output_interface.h"
 #include "shmclientbuffer.h"
@@ -216,69 +215,4 @@ ClientConnection *Display::createClient(int fd)
     return getConnection(c);
 }
 
-struct ClientBufferDestroyListener : wl_listener
-{
-    ClientBufferDestroyListener(Display *display, ClientBuffer *buffer);
-    ~ClientBufferDestroyListener();
-
-    Display *display;
-};
-
-void bufferDestroyCallback(wl_listener *listener, void *data)
-{
-    ClientBufferDestroyListener *destroyListener = static_cast<ClientBufferDestroyListener *>(listener);
-    DisplayPrivate *displayPrivate = DisplayPrivate::get(destroyListener->display);
-
-    ClientBuffer *buffer = displayPrivate->q->clientBufferForResource(static_cast<wl_resource *>(data));
-    displayPrivate->unregisterClientBuffer(buffer);
-
-    buffer->drop();
-}
-
-ClientBufferDestroyListener::ClientBufferDestroyListener(Display *display, ClientBuffer *buffer)
-    : display(display)
-{
-    notify = bufferDestroyCallback;
-
-    link.prev = nullptr;
-    link.next = nullptr;
-
-    wl_resource_add_destroy_listener(buffer->resource(), this);
-}
-
-ClientBufferDestroyListener::~ClientBufferDestroyListener()
-{
-    wl_list_remove(&link);
-}
-
-ClientBuffer *Display::clientBufferForResource(wl_resource *resource) const
-{
-    ClientBuffer *buffer = d->resourceToBuffer.value(resource);
-    if (buffer) {
-        return buffer;
-    }
-
-    for (ClientBufferIntegration *integration : std::as_const(d->bufferIntegrations)) {
-        ClientBuffer *buffer = integration->createBuffer(resource);
-        if (buffer) {
-            d->registerClientBuffer(buffer);
-            return buffer;
-        }
-    }
-    return nullptr;
-}
-
-void DisplayPrivate::registerClientBuffer(ClientBuffer *buffer)
-{
-    resourceToBuffer.insert(buffer->resource(), buffer);
-    bufferToListener.insert(buffer, new ClientBufferDestroyListener(q, buffer));
-}
-
-void DisplayPrivate::unregisterClientBuffer(ClientBuffer *buffer)
-{
-    Q_ASSERT_X(buffer->resource(), "unregisterClientBuffer", "buffer must have valid resource");
-    resourceToBuffer.remove(buffer->resource());
-    delete bufferToListener.take(buffer);
-}
-
-}
+} // namespace KWaylandServer
diff --git a/src/wayland/display.h b/src/wayland/display.h
index 7a3f9a341e..857d11358a 100644
--- a/src/wayland/display.h
+++ b/src/wayland/display.h
@@ -125,12 +125,6 @@ public:
     ClientConnection *getConnection(wl_client *client);
     QVector<ClientConnection *> connections() const;
 
-    /**
-     * Returns the client buffer with the specified @a resource. Returns @c null if there's
-     * no such a buffer.
-     */
-    ClientBuffer *clientBufferForResource(wl_resource *resource) const;
-
 private Q_SLOTS:
     void flush();
 
diff --git a/src/wayland/display_p.h b/src/wayland/display_p.h
index efdf9ab631..816e3cd154 100644
--- a/src/wayland/display_p.h
+++ b/src/wayland/display_p.h
@@ -9,7 +9,6 @@
 
 #include <wayland-server-core.h>
 
-#include <QHash>
 #include <QList>
 #include <QSocketNotifier>
 #include <QString>
@@ -19,14 +18,11 @@ struct wl_resource;
 
 namespace KWaylandServer
 {
-class ClientBufferIntegration;
-class ClientBuffer;
 class ClientConnection;
 class Display;
 class OutputInterface;
 class OutputDeviceV2Interface;
 class SeatInterface;
-struct ClientBufferDestroyListener;
 
 class DisplayPrivate
 {
@@ -36,9 +32,6 @@ public:
 
     void registerSocketName(const QString &socketName);
 
-    void registerClientBuffer(ClientBuffer *clientBuffer);
-    void unregisterClientBuffer(ClientBuffer *clientBuffer);
-
     Display *q;
     QSocketNotifier *socketNotifier = nullptr;
     wl_display *display = nullptr;
@@ -49,9 +42,6 @@ public:
     QVector<SeatInterface *> seats;
     QVector<ClientConnection *> clients;
     QStringList socketNames;
-    QHash<::wl_resource *, ClientBuffer *> resourceToBuffer;
-    QHash<ClientBuffer *, ClientBufferDestroyListener *> bufferToListener;
-    QList<ClientBufferIntegration *> bufferIntegrations;
 };
 
 } // namespace KWaylandServer
diff --git a/src/wayland/drmclientbuffer.cpp b/src/wayland/drmclientbuffer.cpp
index f7d67a3ff2..41bc9574b9 100644
--- a/src/wayland/drmclientbuffer.cpp
+++ b/src/wayland/drmclientbuffer.cpp
@@ -119,7 +119,7 @@ void DrmClientBufferIntegrationPrivate::drm_create_prime_buffer(Resource *resour
 }
 
 DrmClientBufferIntegration::DrmClientBufferIntegration(Display *display)
-    : ClientBufferIntegration(display)
+    : QObject(display)
     , d(std::make_unique<DrmClientBufferIntegrationPrivate>(display))
 {
 }
diff --git a/src/wayland/drmclientbuffer.h b/src/wayland/drmclientbuffer.h
index 8fd877f203..dc641c5f1d 100644
--- a/src/wayland/drmclientbuffer.h
+++ b/src/wayland/drmclientbuffer.h
@@ -6,10 +6,14 @@
 
 #pragma once
 
-#include "clientbufferintegration.h"
+#include "kwin_export.h"
+
+#include <QObject>
 
 namespace KWaylandServer
 {
+
+class Display;
 class DrmClientBufferIntegrationPrivate;
 
 /**
@@ -23,7 +27,7 @@ class DrmClientBufferIntegrationPrivate;
  * Once the wl_drm protocol is no longer mandatory in Xwayland, this stub can be
  * dropped.
  */
-class KWIN_EXPORT DrmClientBufferIntegration : public ClientBufferIntegration
+class KWIN_EXPORT DrmClientBufferIntegration : public QObject
 {
     Q_OBJECT
 
diff --git a/src/wayland/linuxdmabufv1clientbuffer.cpp b/src/wayland/linuxdmabufv1clientbuffer.cpp
index 46194916e9..6e80e2a294 100644
--- a/src/wayland/linuxdmabufv1clientbuffer.cpp
+++ b/src/wayland/linuxdmabufv1clientbuffer.cpp
@@ -160,23 +160,22 @@ void LinuxDmaBufParamsV1::zwp_linux_buffer_params_v1_create(Resource *resource,
     m_attrs.height = height;
     m_attrs.format = format;
 
-    auto clientBuffer = std::make_unique<LinuxDmaBufV1ClientBuffer>(std::move(m_attrs));
-    if (!renderBackend->testImportBuffer(clientBuffer.get())) {
+    auto clientBuffer = new LinuxDmaBufV1ClientBuffer(std::move(m_attrs));
+    if (!renderBackend->testImportBuffer(clientBuffer)) {
         send_failed(resource->handle);
+        delete clientBuffer;
         return;
     }
 
     wl_resource *bufferResource = wl_resource_create(resource->client(), &wl_buffer_interface, 1, 0);
     if (!bufferResource) {
         wl_resource_post_no_memory(resource->handle);
+        delete clientBuffer;
         return;
     }
 
     clientBuffer->initialize(bufferResource);
     send_created(resource->handle, bufferResource);
-
-    DisplayPrivate *displayPrivate = DisplayPrivate::get(m_integration->display());
-    displayPrivate->registerClientBuffer(clientBuffer.release());
 }
 
 void LinuxDmaBufParamsV1::zwp_linux_buffer_params_v1_create_immed(Resource *resource,
@@ -212,22 +211,21 @@ void LinuxDmaBufParamsV1::zwp_linux_buffer_params_v1_create_immed(Resource *reso
     m_attrs.height = height;
     m_attrs.format = format;
 
-    auto clientBuffer = std::make_unique<LinuxDmaBufV1ClientBuffer>(std::move(m_attrs));
-    if (!renderBackend->testImportBuffer(clientBuffer.get())) {
+    auto clientBuffer = new LinuxDmaBufV1ClientBuffer(std::move(m_attrs));
+    if (!renderBackend->testImportBuffer(clientBuffer)) {
         wl_resource_post_error(resource->handle, error_invalid_wl_buffer, "importing the supplied dmabufs failed");
+        delete clientBuffer;
         return;
     }
 
     wl_resource *bufferResource = wl_resource_create(resource->client(), &wl_buffer_interface, 1, buffer_id);
     if (!bufferResource) {
         wl_resource_post_no_memory(resource->handle);
+        delete clientBuffer;
         return;
     }
 
     clientBuffer->initialize(bufferResource);
-
-    DisplayPrivate *displayPrivate = DisplayPrivate::get(m_integration->display());
-    displayPrivate->registerClientBuffer(clientBuffer.release());
 }
 
 bool LinuxDmaBufParamsV1::test(Resource *resource, uint32_t width, uint32_t height)
@@ -291,7 +289,7 @@ bool LinuxDmaBufParamsV1::test(Resource *resource, uint32_t width, uint32_t heig
 }
 
 LinuxDmaBufV1ClientBufferIntegration::LinuxDmaBufV1ClientBufferIntegration(Display *display)
-    : ClientBufferIntegration(display)
+    : QObject(display)
     , d(new LinuxDmaBufV1ClientBufferIntegrationPrivate(this, display))
 {
 }
@@ -366,44 +364,60 @@ static bool testAlphaChannel(uint32_t drmFormat)
     }
 }
 
-void LinuxDmaBufV1ClientBufferPrivate::buffer_destroy(Resource *resource)
+void LinuxDmaBufV1ClientBuffer::buffer_destroy_resource(wl_resource *resource)
 {
-    wl_resource_destroy(resource->handle);
+    if (LinuxDmaBufV1ClientBuffer *buffer = LinuxDmaBufV1ClientBuffer::get(resource)) {
+        buffer->m_resource = nullptr;
+        buffer->drop();
+    }
 }
 
+void LinuxDmaBufV1ClientBuffer::buffer_destroy(wl_client *client, wl_resource *resource)
+{
+    wl_resource_destroy(resource);
+}
+
+const struct wl_buffer_interface LinuxDmaBufV1ClientBuffer::implementation = {
+    .destroy = buffer_destroy,
+};
+
 LinuxDmaBufV1ClientBuffer::LinuxDmaBufV1ClientBuffer(KWin::DmaBufAttributes &&attrs)
-    : ClientBuffer(std::make_unique<LinuxDmaBufV1ClientBufferPrivate>())
 {
-    Q_D(LinuxDmaBufV1ClientBuffer);
-    d->attrs = std::move(attrs);
-    d->hasAlphaChannel = testAlphaChannel(attrs.format);
+    m_attrs = std::move(attrs);
+    m_hasAlphaChannel = testAlphaChannel(m_attrs.format);
 }
 
-LinuxDmaBufV1ClientBuffer::~LinuxDmaBufV1ClientBuffer() = default;
-
 void LinuxDmaBufV1ClientBuffer::initialize(wl_resource *resource)
 {
-    Q_D(LinuxDmaBufV1ClientBuffer);
-    d->init(resource);
-    ClientBuffer::initialize(resource);
+    m_resource = resource;
+    wl_resource_set_implementation(resource, &implementation, this, buffer_destroy_resource);
+
+    connect(this, &GraphicsBuffer::released, [this]() {
+        wl_buffer_send_release(m_resource);
+    });
 }
 
 const KWin::DmaBufAttributes *LinuxDmaBufV1ClientBuffer::dmabufAttributes() const
 {
-    Q_D(const LinuxDmaBufV1ClientBuffer);
-    return &d->attrs;
+    return &m_attrs;
 }
 
 QSize LinuxDmaBufV1ClientBuffer::size() const
 {
-    Q_D(const LinuxDmaBufV1ClientBuffer);
-    return QSize(d->attrs.width, d->attrs.height);
+    return QSize(m_attrs.width, m_attrs.height);
 }
 
 bool LinuxDmaBufV1ClientBuffer::hasAlphaChannel() const
 {
-    Q_D(const LinuxDmaBufV1ClientBuffer);
-    return d->hasAlphaChannel;
+    return m_hasAlphaChannel;
+}
+
+LinuxDmaBufV1ClientBuffer *LinuxDmaBufV1ClientBuffer::get(wl_resource *resource)
+{
+    if (wl_resource_instance_of(resource, &wl_buffer_interface, &implementation)) {
+        return static_cast<LinuxDmaBufV1ClientBuffer *>(wl_resource_get_user_data(resource));
+    }
+    return nullptr;
 }
 
 LinuxDmaBufV1Feedback::LinuxDmaBufV1Feedback(LinuxDmaBufV1ClientBufferIntegrationPrivate *integration)
diff --git a/src/wayland/linuxdmabufv1clientbuffer.h b/src/wayland/linuxdmabufv1clientbuffer.h
index 0d72e8df11..945ac5d260 100644
--- a/src/wayland/linuxdmabufv1clientbuffer.h
+++ b/src/wayland/linuxdmabufv1clientbuffer.h
@@ -9,11 +9,11 @@
 #pragma once
 
 #include "clientbuffer.h"
-#include "clientbufferintegration.h"
 
 #include <QHash>
 #include <QSet>
 #include <sys/types.h>
+#include <wayland-server.h>
 
 namespace KWin
 {
@@ -22,6 +22,8 @@ class RenderBackend;
 
 namespace KWaylandServer
 {
+
+class Display;
 class LinuxDmaBufV1ClientBufferPrivate;
 class LinuxDmaBufV1ClientBufferIntegrationPrivate;
 class LinuxDmaBufV1FeedbackPrivate;
@@ -35,18 +37,27 @@ class LinuxDmaBufV1FeedbackPrivate;
 class KWIN_EXPORT LinuxDmaBufV1ClientBuffer : public ClientBuffer
 {
     Q_OBJECT
-    Q_DECLARE_PRIVATE(LinuxDmaBufV1ClientBuffer)
 
 public:
     LinuxDmaBufV1ClientBuffer(KWin::DmaBufAttributes &&attrs);
-    ~LinuxDmaBufV1ClientBuffer() override;
 
     QSize size() const override;
     bool hasAlphaChannel() const override;
     const KWin::DmaBufAttributes *dmabufAttributes() const override;
 
+    static LinuxDmaBufV1ClientBuffer *get(wl_resource *resource);
+
 private:
     void initialize(wl_resource *resource);
+
+    static void buffer_destroy_resource(wl_resource *resource);
+    static void buffer_destroy(wl_client *client, wl_resource *resource);
+    static const struct wl_buffer_interface implementation;
+
+    wl_resource *m_resource = nullptr;
+    KWin::DmaBufAttributes m_attrs;
+    bool m_hasAlphaChannel = false;
+
     friend class LinuxDmaBufParamsV1;
 };
 
@@ -84,7 +95,7 @@ private:
 /**
  * The LinuxDmaBufV1ClientBufferIntegration class provides support for linux dma-buf buffers.
  */
-class KWIN_EXPORT LinuxDmaBufV1ClientBufferIntegration : public ClientBufferIntegration
+class KWIN_EXPORT LinuxDmaBufV1ClientBufferIntegration : public QObject
 {
     Q_OBJECT
 
diff --git a/src/wayland/linuxdmabufv1clientbuffer_p.h b/src/wayland/linuxdmabufv1clientbuffer_p.h
index 9eb23529c2..17dc5d7e7c 100644
--- a/src/wayland/linuxdmabufv1clientbuffer_p.h
+++ b/src/wayland/linuxdmabufv1clientbuffer_p.h
@@ -10,16 +10,15 @@
     SPDX-License-Identifier: LGPL-2.1-only OR LGPL-3.0-only OR LicenseRef-KDE-Accepted-LGPL
 */
 #pragma once
-#include "clientbuffer_p.h"
+
 #include "display.h"
-#include "display_p.h"
 #include "linuxdmabufv1clientbuffer.h"
 #include "utils/ramfile.h"
 
 #include "qwayland-server-linux-dmabuf-unstable-v1.h"
-#include "qwayland-server-wayland.h"
 
 #include <QDebug>
+#include <QPointer>
 #include <QVector>
 
 #include <drm_fourcc.h>
@@ -49,16 +48,6 @@ protected:
     void zwp_linux_dmabuf_v1_get_surface_feedback(Resource *resource, uint32_t id, wl_resource *surface) override;
 };
 
-class LinuxDmaBufV1ClientBufferPrivate : public ClientBufferPrivate, public QtWaylandServer::wl_buffer
-{
-public:
-    KWin::DmaBufAttributes attrs;
-    bool hasAlphaChannel = false;
-
-protected:
-    void buffer_destroy(Resource *resource) override;
-};
-
 class LinuxDmaBufParamsV1 : public QtWaylandServer::zwp_linux_buffer_params_v1
 {
 public:
diff --git a/src/wayland/shadow_interface.cpp b/src/wayland/shadow_interface.cpp
index b2cb1a3628..fbe2ab231b 100644
--- a/src/wayland/shadow_interface.cpp
+++ b/src/wayland/shadow_interface.cpp
@@ -20,7 +20,6 @@ public:
     ShadowManagerInterfacePrivate(ShadowManagerInterface *_q, Display *display);
 
     ShadowManagerInterface *q;
-    Display *display;
 
 protected:
     void org_kde_kwin_shadow_manager_create(Resource *resource, uint32_t id, wl_resource *surface) override;
@@ -31,7 +30,6 @@ protected:
 ShadowManagerInterfacePrivate::ShadowManagerInterfacePrivate(ShadowManagerInterface *_q, Display *display)
     : QtWaylandServer::org_kde_kwin_shadow_manager(*display, s_version)
     , q(_q)
-    , display(display)
 {
 }
 
@@ -79,11 +77,6 @@ ShadowManagerInterface::ShadowManagerInterface(Display *display, QObject *parent
 
 ShadowManagerInterface::~ShadowManagerInterface() = default;
 
-Display *ShadowManagerInterface::display() const
-{
-    return d->display;
-}
-
 class ShadowInterfacePrivate : public QtWaylandServer::org_kde_kwin_shadow
 {
 public:
@@ -172,7 +165,7 @@ void ShadowInterfacePrivate::org_kde_kwin_shadow_commit(Resource *resource)
 
 void ShadowInterfacePrivate::attach(ShadowInterfacePrivate::State::Flags flag, wl_resource *buffer)
 {
-    ClientBuffer *b = manager->display()->clientBufferForResource(buffer);
+    ClientBuffer *b = ClientBuffer::get(buffer);
     switch (flag) {
     case State::LeftBuffer:
         pending.left = b;
diff --git a/src/wayland/shadow_interface.h b/src/wayland/shadow_interface.h
index 4a4058426a..49105ae669 100644
--- a/src/wayland/shadow_interface.h
+++ b/src/wayland/shadow_interface.h
@@ -28,8 +28,6 @@ public:
     explicit ShadowManagerInterface(Display *display, QObject *parent = nullptr);
     ~ShadowManagerInterface() override;
 
-    Display *display() const;
-
 private:
     std::unique_ptr<ShadowManagerInterfacePrivate> d;
 };
diff --git a/src/wayland/shmclientbuffer.cpp b/src/wayland/shmclientbuffer.cpp
index 81689b6d5b..d02e8bc27e 100644
--- a/src/wayland/shmclientbuffer.cpp
+++ b/src/wayland/shmclientbuffer.cpp
@@ -5,9 +5,11 @@
 */
 
 #include "shmclientbuffer.h"
-#include "clientbuffer_p.h"
 #include "display.h"
 
+#include <QHash>
+#include <QImage>
+
 #include <wayland-server-core.h>
 #include <wayland-server-protocol.h>
 
@@ -15,8 +17,9 @@ namespace KWaylandServer
 {
 static const ShmClientBuffer *s_accessedBuffer = nullptr;
 static int s_accessCounter = 0;
+static QHash<wl_resource *, ShmClientBuffer *> s_buffers;
 
-class ShmClientBufferPrivate : public ClientBufferPrivate
+class ShmClientBufferPrivate
 {
 public:
     ShmClientBufferPrivate(ShmClientBuffer *q);
@@ -24,6 +27,7 @@ public:
     static void buffer_destroy_callback(wl_listener *listener, void *data);
 
     ShmClientBuffer *q;
+    wl_resource *resource = nullptr;
     QImage::Format format = QImage::Format_Invalid;
     uint32_t width = 0;
     uint32_t height = 0;
@@ -51,12 +55,13 @@ static void cleanupShmPool(void *poolHandle)
 void ShmClientBufferPrivate::buffer_destroy_callback(wl_listener *listener, void *data)
 {
     auto bufferPrivate = reinterpret_cast<ShmClientBufferPrivate::DestroyListener *>(listener)->receiver;
-    wl_shm_buffer *buffer = wl_shm_buffer_get(bufferPrivate->q->resource());
+    wl_shm_buffer *buffer = wl_shm_buffer_get(bufferPrivate->resource);
     wl_shm_pool *pool = wl_shm_buffer_ref_pool(buffer);
 
     wl_list_remove(&bufferPrivate->destroyListener.listener.link);
     wl_list_init(&bufferPrivate->destroyListener.listener.link);
 
+    bufferPrivate->resource = nullptr;
     bufferPrivate->savedData = QImage(static_cast<const uchar *>(wl_shm_buffer_get_data(buffer)),
                                       bufferPrivate->width,
                                       bufferPrivate->height,
@@ -64,6 +69,8 @@ void ShmClientBufferPrivate::buffer_destroy_callback(wl_listener *listener, void
                                       bufferPrivate->format,
                                       cleanupShmPool,
                                       pool);
+
+    bufferPrivate->q->drop();
 }
 
 static bool alphaChannelFromFormat(uint32_t format)
@@ -110,32 +117,35 @@ static QImage::Format imageFormatForShmFormat(uint32_t format)
 }
 
 ShmClientBuffer::ShmClientBuffer(wl_resource *resource)
-    : ClientBuffer(resource, std::make_unique<ShmClientBufferPrivate>(this))
+    : d(std::make_unique<ShmClientBufferPrivate>(this))
 {
-    Q_D(ShmClientBuffer);
-
     wl_shm_buffer *buffer = wl_shm_buffer_get(resource);
+    d->resource = resource;
     d->width = wl_shm_buffer_get_width(buffer);
     d->height = wl_shm_buffer_get_height(buffer);
     d->hasAlphaChannel = alphaChannelFromFormat(wl_shm_buffer_get_format(buffer));
     d->format = imageFormatForShmFormat(wl_shm_buffer_get_format(buffer));
 
-    // The underlying shm pool will be referenced if the wl_shm_buffer is destroyed so the
-    // compositor can access buffer data even after the buffer is gone.
-    d->destroyListener.receiver = d;
+    d->destroyListener.receiver = d.get();
     d->destroyListener.listener.notify = ShmClientBufferPrivate::buffer_destroy_callback;
     wl_resource_add_destroy_listener(resource, &d->destroyListener.listener);
+
+    connect(this, &GraphicsBuffer::released, [this]() {
+        wl_buffer_send_release(d->resource);
+    });
+}
+
+ShmClientBuffer::~ShmClientBuffer()
+{
 }
 
 QSize ShmClientBuffer::size() const
 {
-    Q_D(const ShmClientBuffer);
     return QSize(d->width, d->height);
 }
 
 bool ShmClientBuffer::hasAlphaChannel() const
 {
-    Q_D(const ShmClientBuffer);
     return d->hasAlphaChannel;
 }
 
@@ -155,8 +165,7 @@ QImage ShmClientBuffer::data() const
         return QImage();
     }
 
-    Q_D(const ShmClientBuffer);
-    if (wl_shm_buffer *buffer = wl_shm_buffer_get(resource())) {
+    if (wl_shm_buffer *buffer = wl_shm_buffer_get(d->resource)) {
         s_accessedBuffer = this;
         s_accessCounter++;
         wl_shm_buffer_begin_access(buffer);
@@ -167,8 +176,26 @@ QImage ShmClientBuffer::data() const
     return d->savedData;
 }
 
+ShmClientBuffer *ShmClientBuffer::get(wl_resource *resource)
+{
+    if (auto buffer = s_buffers.value(resource)) {
+        return buffer;
+    }
+
+    if (wl_shm_buffer_get(resource)) {
+        auto buffer = new ShmClientBuffer(resource);
+        s_buffers[resource] = buffer;
+        connect(buffer, &ShmClientBuffer::dropped, [resource]() {
+            s_buffers.remove(resource);
+        });
+        return buffer;
+    }
+
+    return nullptr;
+}
+
 ShmClientBufferIntegration::ShmClientBufferIntegration(Display *display)
-    : ClientBufferIntegration(display)
+    : QObject(display)
 {
 #if Q_BYTE_ORDER == Q_LITTLE_ENDIAN
     wl_display_add_shm_format(*display, WL_SHM_FORMAT_ARGB2101010);
@@ -181,12 +208,4 @@ ShmClientBufferIntegration::ShmClientBufferIntegration(Display *display)
     wl_display_init_shm(*display);
 }
 
-ClientBuffer *ShmClientBufferIntegration::createBuffer(::wl_resource *resource)
-{
-    if (wl_shm_buffer_get(resource)) {
-        return new ShmClientBuffer(resource);
-    }
-    return nullptr;
-}
-
 } // namespace KWaylandServer
diff --git a/src/wayland/shmclientbuffer.h b/src/wayland/shmclientbuffer.h
index d7695a06b2..eed64ad81b 100644
--- a/src/wayland/shmclientbuffer.h
+++ b/src/wayland/shmclientbuffer.h
@@ -7,10 +7,11 @@
 #pragma once
 
 #include "clientbuffer.h"
-#include "clientbufferintegration.h"
 
 namespace KWaylandServer
 {
+
+class Display;
 class ShmClientBufferPrivate;
 
 /**
@@ -22,28 +23,31 @@ class ShmClientBufferPrivate;
 class KWIN_EXPORT ShmClientBuffer : public ClientBuffer
 {
     Q_OBJECT
-    Q_DECLARE_PRIVATE(ShmClientBuffer)
 
 public:
     explicit ShmClientBuffer(wl_resource *resource);
+    ~ShmClientBuffer() override;
 
     QImage data() const;
 
     QSize size() const override;
     bool hasAlphaChannel() const override;
+
+    static ShmClientBuffer *get(wl_resource *resource);
+
+private:
+    std::unique_ptr<ShmClientBufferPrivate> d;
 };
 
 /**
  * The ShmClientBufferIntegration class provides support for wl_shm_buffer buffers.
  */
-class ShmClientBufferIntegration : public ClientBufferIntegration
+class ShmClientBufferIntegration : public QObject
 {
     Q_OBJECT
 
 public:
     explicit ShmClientBufferIntegration(Display *display);
-
-    ClientBuffer *createBuffer(::wl_resource *resource) override;
 };
 
 } // namespace KWaylandServer
diff --git a/src/wayland/surface_interface.cpp b/src/wayland/surface_interface.cpp
index 7ed37cbea2..a80a959fc7 100644
--- a/src/wayland/surface_interface.cpp
+++ b/src/wayland/surface_interface.cpp
@@ -270,7 +270,7 @@ void SurfaceInterfacePrivate::surface_attach(Resource *resource, struct ::wl_res
         pending.bufferDamage = QRegion();
         return;
     }
-    pending.buffer = compositor->display()->clientBufferForResource(buffer);
+    pending.buffer = ClientBuffer::get(buffer);
 }
 
 void SurfaceInterfacePrivate::surface_damage(Resource *, int32_t x, int32_t y, int32_t width, int32_t height)