Move the loadTexture for X11 pixmap functionality to the EglOnXBackend

Summary:
To use eglCreateImageKhr for an X11 pixmap we need an EGLDisplay created
for the same XDisplay as the X11 pixmap. This means if we created an
EGLDisplay for a GBM device, we are not allowed to load a texture from
the X11 pixmap and can result in a crash in the driver.

Similar in the nested X11 setup the EGLDisplay is created for the
rendering window, but the X11 pixmaps are from the Xwayland server KWin
started. They don't belong to the same windowing system.

This change addresses this problem by moving the loading of X11 pixmaps
from AbstractEglTexture to EglTexture of the EglOnX11Backend. Thus for
any usage on a non X11 platform we cannot hit the code path any more.
In addition the nested X11 platform can indicate that it doesn't support
it and thus also doesn't go through the code path.

Test Plan: Tested standalone and nested X11 platform

Reviewers: #kwin, #plasma_on_wayland

Subscribers: plasma-devel, kwin

Tags: #plasma_on_wayland, #kwin

Differential Revision: https://phabricator.kde.org/D1857

abstract_egl_backend.cpp

@@ -286,8 +286,7 @@ bool AbstractEglTexture::loadTexture(WindowPixmap *pixmap)
         if (updateFromFBO(pixmap->fbo())) {
             return true;
         }
-        // try X11 loading
-        return loadTexture(pixmap->pixmap(), pixmap->toplevel()->size());
+        return false;
     }
     // try Wayland loading
     if (auto s = pixmap->surface()) {
@@ -300,36 +299,6 @@ bool AbstractEglTexture::loadTexture(WindowPixmap *pixmap)
     }
 }
 
-bool AbstractEglTexture::loadTexture(xcb_pixmap_t pix, const QSize &size)
-{
-    if (pix == XCB_NONE)
-        return false;
-
-    glGenTextures(1, &m_texture);
-    q->setWrapMode(GL_CLAMP_TO_EDGE);
-    q->setFilter(GL_LINEAR);
-    q->bind();
-    const EGLint attribs[] = {
-        EGL_IMAGE_PRESERVED_KHR, EGL_TRUE,
-        EGL_NONE
-    };
-    m_image = eglCreateImageKHR(m_backend->eglDisplay(), EGL_NO_CONTEXT, EGL_NATIVE_PIXMAP_KHR,
-                                          (EGLClientBuffer)pix, attribs);
-
-    if (EGL_NO_IMAGE_KHR == m_image) {
-        qCDebug(KWIN_CORE) << "failed to create egl image";
-        q->unbind();
-        q->discard();
-        return false;
-    }
-    glEGLImageTargetTexture2DOES(GL_TEXTURE_2D, (GLeglImageOES)m_image);
-    q->unbind();
-    q->setYInverted(true);
-    m_size = size;
-    updateMatrix();
-    return true;
-}
-
 void AbstractEglTexture::updateTexture(WindowPixmap *pixmap)
 {
     const auto &buffer = pixmap->buffer();

abstract_egl_backend.h

@@ -90,9 +90,14 @@ protected:
     EGLImageKHR image() const {
         return m_image;
     }
+    void setImage(const EGLImageKHR &img) {
+        m_image = img;
+    }
+    SceneOpenGL::Texture *texture() const {
+        return q;
+    }
 
 private:
-    bool loadTexture(xcb_pixmap_t pix, const QSize &size);
     bool loadShmTexture(const QPointer<KWayland::Server::BufferInterface> &buffer);
     bool loadEglTexture(const QPointer<KWayland::Server::BufferInterface> &buffer);
     EGLImageKHR attach(const QPointer<KWayland::Server::BufferInterface> &buffer);

plugins/platforms/x11/common/eglonxbackend.cpp

@@ -469,11 +469,57 @@ bool EglOnXBackend::makeContextCurrent(const EGLSurface &surface)
 
 EglTexture::EglTexture(KWin::SceneOpenGL::Texture *texture, KWin::EglOnXBackend *backend)
     : AbstractEglTexture(texture, backend)
+    , m_backend(backend)
 {
 }
 
 EglTexture::~EglTexture() = default;
 
+bool EglTexture::loadTexture(WindowPixmap *pixmap)
+{
+    // first try the Wayland enabled loading
+    if (AbstractEglTexture::loadTexture(pixmap)) {
+        return true;
+    }
+    // did not succeed, try on X11
+    return loadTexture(pixmap->pixmap(), pixmap->toplevel()->size());
+}
+
+bool EglTexture::loadTexture(xcb_pixmap_t pix, const QSize &size)
+{
+    if (!m_backend->isX11TextureFromPixmapSupported()) {
+        return false;
+    }
+
+    if (pix == XCB_NONE)
+        return false;
+
+    glGenTextures(1, &m_texture);
+    auto q = texture();
+    q->setWrapMode(GL_CLAMP_TO_EDGE);
+    q->setFilter(GL_LINEAR);
+    q->bind();
+    const EGLint attribs[] = {
+        EGL_IMAGE_PRESERVED_KHR, EGL_TRUE,
+        EGL_NONE
+    };
+    setImage(eglCreateImageKHR(m_backend->eglDisplay(), EGL_NO_CONTEXT, EGL_NATIVE_PIXMAP_KHR,
+                               (EGLClientBuffer)pix, attribs));
+
+    if (EGL_NO_IMAGE_KHR == image()) {
+        qCDebug(KWIN_CORE) << "failed to create egl image";
+        q->unbind();
+        q->discard();
+        return false;
+    }
+    glEGLImageTargetTexture2DOES(GL_TEXTURE_2D, (GLeglImageOES)image());
+    q->unbind();
+    q->setYInverted(true);
+    m_size = size;
+    updateMatrix();
+    return true;
+}
+
 void KWin::EglTexture::onDamage()
 {
     if (options->isGlStrictBinding()) {

plugins/platforms/x11/common/eglonxbackend.h

@@ -42,6 +42,10 @@ public:
     virtual bool usesOverlayWindow() const override;
     void init() override;
 
+    bool isX11TextureFromPixmapSupported() const {
+        return m_x11TextureFromPixmapSupported;
+    }
+
 protected:
     virtual void present();
     void presentSurface(EGLSurface surface, const QRegion &damage, const QRect &screenGeometry);
@@ -55,6 +59,10 @@ protected:
     }
     bool makeContextCurrent(const EGLSurface &surface);
 
+    void setX11TextureFromPixmapSupported(bool set) {
+        m_x11TextureFromPixmapSupported = set;
+    }
+
 private:
     bool initBufferConfigs();
     bool initRenderingContext();
@@ -71,6 +79,7 @@ private:
     int m_x11ScreenNumber;
     xcb_window_t m_renderingWindow = XCB_WINDOW_NONE;
     bool m_havePlatformBase = false;
+    bool m_x11TextureFromPixmapSupported = true;
     friend class EglTexture;
 };
 
@@ -82,10 +91,13 @@ class EglTexture : public AbstractEglTexture
 public:
     virtual ~EglTexture();
     virtual void onDamage();
+    bool loadTexture(WindowPixmap *pixmap) override;
 
 private:
+    bool loadTexture(xcb_pixmap_t pix, const QSize &size);
     friend class EglOnXBackend;
     EglTexture(SceneOpenGL::Texture *texture, EglOnXBackend *backend);
+    EglOnXBackend *m_backend;
 };
 
 } // namespace

plugins/platforms/x11/windowed/egl_x11_backend.cpp

@@ -31,6 +31,7 @@ EglX11Backend::EglX11Backend(X11WindowedBackend *backend)
     : EglOnXBackend(backend->connection(), backend->display(), backend->rootWindow(), backend->screenNumer(), XCB_WINDOW_NONE)
     , m_backend(backend)
 {
+    setX11TextureFromPixmapSupported(false);
 }
 
 EglX11Backend::~EglX11Backend() = default;