diff --git a/input.cpp b/input.cpp
index faa4dfc881..8aa4b99996 100644
--- a/input.cpp
+++ b/input.cpp
@@ -33,6 +33,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #include "libinput/connection.h"
 #endif
 #if HAVE_WAYLAND
+#include "wayland_backend.h"
 #include "wayland_server.h"
 #include <KWayland/Server/seat_interface.h>
 #endif
@@ -267,10 +268,17 @@ void InputRedirection::updatePointerWindow()
         // disconnect old surface
         if (oldWindow) {
             disconnect(oldWindow.data(), &Toplevel::geometryChanged, this, &InputRedirection::updateFocusedPointerPosition);
+            if (Wayland::WaylandBackend *w = Wayland::WaylandBackend::self()) {
+                disconnect(seat->focusedPointer()->cursor(), &KWayland::Server::Cursor::changed, w, &Wayland::WaylandBackend::installCursorFromServer);
+            }
         }
         if (t && t->surface()) {
             seat->setFocusedPointerSurface(t->surface(), t->pos());
             connect(t, &Toplevel::geometryChanged, this, &InputRedirection::updateFocusedPointerPosition);
+            if (Wayland::WaylandBackend *w = Wayland::WaylandBackend::self()) {
+                w->installCursorFromServer();
+                connect(seat->focusedPointer()->cursor(), &KWayland::Server::Cursor::changed, w, &Wayland::WaylandBackend::installCursorFromServer);
+            }
         } else {
             seat->setFocusedPointerSurface(nullptr);
             t = nullptr;
diff --git a/wayland_backend.cpp b/wayland_backend.cpp
index 1c36a6a986..2a52aade16 100644
--- a/wayland_backend.cpp
+++ b/wayland_backend.cpp
@@ -41,15 +41,15 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #include <KWayland/Client/subcompositor.h>
 #include <KWayland/Client/subsurface.h>
 #include <KWayland/Client/surface.h>
+#include <KWayland/Server/buffer_interface.h>
 #include <KWayland/Server/seat_interface.h>
+#include <KWayland/Server/surface_interface.h>
 // Qt
 #include <QAbstractEventDispatcher>
 #include <QCoreApplication>
 #include <QDebug>
 #include <QMetaMethod>
 #include <QThread>
-// xcb
-#include <xcb/xfixes.h>
 // Wayland
 #include <wayland-client-protocol.h>
 #include <wayland-cursor.h>
@@ -61,97 +61,6 @@ namespace Wayland
 
 using namespace KWayland::Client;
 
-CursorData::CursorData()
-    : m_valid(init())
-{
-}
-
-CursorData::~CursorData()
-{
-}
-
-bool CursorData::init()
-{
-    QScopedPointer<xcb_xfixes_get_cursor_image_reply_t, QScopedPointerPodDeleter> cursor(
-        xcb_xfixes_get_cursor_image_reply(connection(),
-                                          xcb_xfixes_get_cursor_image_unchecked(connection()),
-                                          NULL));
-    if (cursor.isNull()) {
-        return false;
-    }
-
-    QImage cursorImage((uchar *) xcb_xfixes_get_cursor_image_cursor_image(cursor.data()), cursor->width, cursor->height,
-                       QImage::Format_ARGB32_Premultiplied);
-    if (cursorImage.isNull()) {
-        return false;
-    }
-    // the backend for the cursorImage is destroyed once the xcb cursor goes out of scope
-    // because of that we create a copy
-    m_cursor = cursorImage.copy();
-
-    m_hotSpot = QPoint(cursor->xhot, cursor->yhot);
-    return true;
-}
-
-X11CursorTracker::X11CursorTracker(WaylandBackend *backend, QObject* parent)
-    : QObject(parent)
-    , m_backend(backend)
-    , m_lastX11Cursor(0)
-{
-    Cursor::self()->startCursorTracking();
-    connect(Cursor::self(), SIGNAL(cursorChanged(uint32_t)), SLOT(cursorChanged(uint32_t)));
-}
-
-X11CursorTracker::~X11CursorTracker()
-{
-    if (Cursor::self()) {
-        // Cursor might have been destroyed before Wayland backend gets destroyed
-        Cursor::self()->stopCursorTracking();
-    }
-}
-
-void X11CursorTracker::cursorChanged(uint32_t serial)
-{
-    if (m_lastX11Cursor == serial) {
-        // not changed;
-        return;
-    }
-    m_lastX11Cursor = serial;
-    QHash<uint32_t, CursorData>::iterator it = m_cursors.find(serial);
-    if (it != m_cursors.end()) {
-        installCursor(it.value());
-        return;
-    }
-    ShmPool *pool = m_backend->shmPool();
-    if (!pool->isValid()) {
-        return;
-    }
-    CursorData cursor;
-    if (cursor.isValid()) {
-        // TODO: discard unused cursors after some time?
-        m_cursors.insert(serial, cursor);
-    }
-    installCursor(cursor);
-}
-
-void X11CursorTracker::installCursor(const CursorData& cursor)
-{
-    const QImage &cursorImage = cursor.cursor();
-    auto buffer = m_backend->shmPool()->createBuffer(cursorImage);
-    if (!buffer) {
-        return;
-    }
-    emit cursorImageChanged(buffer, cursorImage.size(), cursor.hotSpot());
-}
-
-void X11CursorTracker::resetCursor()
-{
-    QHash<uint32_t, CursorData>::iterator it = m_cursors.find(m_lastX11Cursor);
-    if (it != m_cursors.end()) {
-        installCursor(it.value());
-    }
-}
-
 WaylandSeat::WaylandSeat(wl_seat *seat, WaylandBackend *backend)
     : QObject(NULL)
     , m_seat(new Seat(this))
@@ -244,14 +153,6 @@ WaylandSeat::WaylandSeat(wl_seat *seat, WaylandBackend *backend)
                         input()->processPointerAxis(toAxis(), delta, time);
                     }
                 );
-                connect(m_backend->cursorTracker(), &X11CursorTracker::cursorImageChanged, this,
-                    [this](Buffer::Ptr image, const QSize &size, const QPoint &hotspot) {
-                        if (image.isNull()) {
-                            return;
-                        }
-                        installCursorImage(image.toStrongRef()->buffer(), size, hotspot);
-                    }
-                );
             } else {
                 destroyPointer();
             }
@@ -317,6 +218,11 @@ void WaylandSeat::installCursorImage(Qt::CursorShape shape)
                        QPoint(image->hotspot_x, image->hotspot_y));
 }
 
+void WaylandSeat::installCursorImage(const QImage &image, const QPoint &hotSpot)
+{
+    installCursorImage(*(m_backend->shmPool()->createBuffer(image).data()), image.size(), hotSpot);
+}
+
 void WaylandSeat::setInstallCursor(bool install)
 {
     // TODO: remove, add?
@@ -381,14 +287,6 @@ WaylandCursor::WaylandCursor(Surface *parentSurface, WaylandBackend *backend)
     auto surface = backend->compositor()->createSurface(this);
     m_subSurface = backend->subCompositor()->createSubSurface(QPointer<Surface>(surface), QPointer<Surface>(parentSurface), this);
 
-    connect(m_backend->cursorTracker(), &X11CursorTracker::cursorImageChanged, this,
-        [this](Buffer::Ptr image, const QSize &size, const QPoint &hotspot) {
-            if (image.isNull()) {
-                return;
-            }
-            setCursorImage(image.toStrongRef()->buffer(), size, hotspot);
-        }
-    );
     connect(Cursor::self(), &Cursor::posChanged, this,
         [this](const QPoint &pos) {
             m_subSurface->setPosition(pos - m_hotSpot);
@@ -432,6 +330,11 @@ void WaylandCursor::setCursorImage(wl_buffer *image, const QSize &size, const QP
     parent->commit(Surface::CommitFlag::None);
 }
 
+void WaylandCursor::setCursorImage(const QImage &image, const QPoint &hotspot)
+{
+    setCursorImage(*(m_backend->shmPool()->createBuffer(image).data()), image.size(), hotspot);
+}
+
 void WaylandCursor::setCursorImage(Qt::CursorShape shape)
 {
     wl_cursor_image *image = m_theme->get(shape);
@@ -462,7 +365,6 @@ WaylandBackend::WaylandBackend(QObject *parent)
     , m_shellSurface(NULL)
     , m_seat()
     , m_shm(new ShmPool(this))
-    , m_cursorTracker()
     , m_connectionThreadObject(nullptr)
     , m_connectionThread(nullptr)
     , m_fullscreenShell(new FullscreenShell(this))
@@ -558,14 +460,12 @@ void WaylandBackend::initConnection()
             // setup registry
             m_registry->create(m_display);
             m_registry->setup();
-            m_cursorTracker.reset(new X11CursorTracker(this, this));
         },
         Qt::QueuedConnection);
     connect(m_connectionThreadObject, &ConnectionThread::connectionDied, this,
         [this]() {
             m_ready = false;
             emit systemCompositorDied();
-            m_cursorTracker.reset();
             m_seat.reset();
             m_shm->destroy();
             destroyOutputs();
@@ -609,6 +509,30 @@ void WaylandBackend::installCursorImage(Qt::CursorShape shape)
     }
 }
 
+void WaylandBackend::installCursorFromServer()
+{
+    if (!waylandServer() || !waylandServer()->seat()->focusedPointer()) {
+        return;
+    }
+    auto c = waylandServer()->seat()->focusedPointer()->cursor();
+    if (c) {
+        auto cursorSurface = c->surface();
+        if (!cursorSurface.isNull()) {
+            auto buffer = cursorSurface.data()->buffer();
+            if (buffer) {
+                // set cursor
+                if (!m_seat.isNull() && m_seat->isInstallCursor()) {
+                    m_seat->installCursorImage(buffer->data(), c->hotspot());
+                } else if (m_cursor) {
+                    m_cursor->setCursorImage(buffer->data(), c->hotspot());
+                }
+                return;
+            }
+        }
+    }
+    // TODO: unset cursor
+}
+
 void WaylandBackend::createSurface()
 {
     m_surface = m_compositor->createSurface(this);
@@ -642,7 +566,6 @@ void WaylandBackend::createSurface()
         // map the surface as fullscreen
         m_shellSurface = m_shell->createSurface(m_surface, this);
         m_shellSurface->setFullscreen();
-        connect(m_shellSurface, &ShellSurface::pinged, m_cursorTracker.data(), &X11CursorTracker::resetCursor);
         connect(m_shellSurface, &ShellSurface::sizeChanged, this, &WaylandBackend::shellSurfaceSizeChanged);
     }
 }
diff --git a/wayland_backend.h b/wayland_backend.h
index f1a6bbcecd..337a9858fb 100644
--- a/wayland_backend.h
+++ b/wayland_backend.h
@@ -68,42 +68,6 @@ namespace Wayland
 class WaylandBackend;
 class WaylandSeat;
 
-class CursorData
-{
-public:
-    CursorData();
-    ~CursorData();
-    bool isValid() const;
-    const QPoint &hotSpot() const;
-    const QImage &cursor() const;
-private:
-    bool init();
-    QImage m_cursor;
-    QPoint m_hotSpot;
-    bool m_valid;
-};
-
-class X11CursorTracker : public QObject
-{
-    Q_OBJECT
-public:
-    explicit X11CursorTracker(WaylandBackend *backend, QObject* parent = 0);
-    virtual ~X11CursorTracker();
-    void resetCursor();
-
-Q_SIGNALS:
-    void cursorImageChanged(QWeakPointer<KWayland::Client::Buffer> image, const QSize &size, const QPoint &hotSpot);
-
-private Q_SLOTS:
-    void cursorChanged(uint32_t serial);
-private:
-    void installCursor(const CursorData &cursor);
-    QHash<uint32_t, CursorData> m_cursors;
-    WaylandBackend *m_backend;
-    uint32_t m_installedCursor;
-    uint32_t m_lastX11Cursor;
-};
-
 class WaylandCursorTheme : public QObject
 {
     Q_OBJECT
@@ -129,6 +93,7 @@ public:
 
     void installCursorImage(wl_buffer *image, const QSize &size, const QPoint &hotspot);
     void installCursorImage(Qt::CursorShape shape);
+    void installCursorImage(const QImage &image, const QPoint &hotspot);
     void setInstallCursor(bool install);
     bool isInstallCursor() const {
         return m_installCursor;
@@ -157,6 +122,7 @@ public:
         return m_hotSpot;
     }
     void setCursorImage(wl_buffer *image, const QSize &size, const QPoint &hotspot);
+    void setCursorImage(const QImage &image, const QPoint &hotspot);
     void setCursorImage(Qt::CursorShape shape);
 
 Q_SIGNALS:
@@ -185,11 +151,11 @@ public:
     const QList<KWayland::Client::Output*> &outputs() const;
     KWayland::Client::ShmPool *shmPool();
     KWayland::Client::SubCompositor *subCompositor();
-    X11CursorTracker *cursorTracker();
 
     KWayland::Client::Surface *surface() const;
     QSize shellSurfaceSize() const;
     void installCursorImage(Qt::CursorShape shape);
+    void installCursorFromServer();
 
 protected:
     void connectNotify(const QMetaMethod &signal) override;
@@ -214,7 +180,6 @@ private:
     KWayland::Client::ShellSurface *m_shellSurface;
     QScopedPointer<WaylandSeat> m_seat;
     KWayland::Client::ShmPool *m_shm;
-    QScopedPointer<X11CursorTracker> m_cursorTracker;
     QList<KWayland::Client::Output*> m_outputs;
     KWayland::Client::ConnectionThread *m_connectionThreadObject;
     QThread *m_connectionThread;
@@ -226,24 +191,6 @@ private:
     KWIN_SINGLETON(WaylandBackend)
 };
 
-inline
-bool CursorData::isValid() const
-{
-    return m_valid;
-}
-
-inline
-const QPoint& CursorData::hotSpot() const
-{
-    return m_hotSpot;
-}
-
-inline
-const QImage &CursorData::cursor() const
-{
-    return m_cursor;
-}
-
 inline
 wl_display *WaylandBackend::display()
 {
@@ -268,12 +215,6 @@ KWayland::Client::ShmPool* WaylandBackend::shmPool()
     return m_shm;
 }
 
-inline
-X11CursorTracker *WaylandBackend::cursorTracker()
-{
-    return m_cursorTracker.data();
-}
-
 inline
 KWayland::Client::Surface *WaylandBackend::surface() const
 {