Composite windows from a QOpenGLFramebufferObject

This change introduces a mechanism for internal windows to be rendered
to a QOpenGLFramebufferObject to be composited using the texture bound
to the FBO. This is useful for in-process rendering (e.g. QtQuick) and
at the same time bypassing the windowing system.

The OpenGL context of the QOpenGLFramebufferObject needs to be sharing
with the compositing OpenGL context.

abstract_egl_backend.cpp

@@ -26,6 +26,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #include <kwinglplatform.h>
 // Qt
 #include <QOpenGLContext>
+#include <QOpenGLFramebufferObject>
 
 namespace KWin
 {
@@ -201,6 +202,9 @@ bool AbstractEglTexture::loadTexture(WindowPixmap *pixmap)
 {
     const auto &buffer = pixmap->buffer();
     if (buffer.isNull()) {
+        if (updateFromFBO(pixmap->fbo())) {
+            return true;
+        }
         // try X11 loading
         return loadTexture(pixmap->pixmap(), pixmap->toplevel()->size());
     }
@@ -246,6 +250,13 @@ void AbstractEglTexture::updateTexture(WindowPixmap *pixmap)
 {
     const auto &buffer = pixmap->buffer();
     if (buffer.isNull()) {
+        const auto &fbo = pixmap->fbo();
+        if (!fbo.isNull()) {
+            if (m_texture != fbo->texture()) {
+                updateFromFBO(fbo);
+            }
+            return;
+        }
         return;
     }
     if (!buffer->shmBuffer()) {
@@ -393,5 +404,19 @@ EGLImageKHR AbstractEglTexture::attach(const QPointer< KWayland::Server::BufferI
     return image;
 }
 
+bool AbstractEglTexture::updateFromFBO(const QSharedPointer<QOpenGLFramebufferObject> &fbo)
+{
+    if (fbo.isNull()) {
+        return false;
+    }
+    m_texture = fbo->texture();
+    m_size = fbo->size();
+    q->setWrapMode(GL_CLAMP_TO_EDGE);
+    q->setFilter(GL_LINEAR);
+    q->setYInverted(false);
+    updateMatrix();
+    return true;
+}
+
 }
 

abstract_egl_backend.h

@@ -21,6 +21,8 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #define KWIN_ABSTRACT_EGL_BACKEND_H
 #include "scene_opengl.h"
 
+class QOpenGLFramebufferObject;
+
 namespace KWin
 {
 
@@ -94,6 +96,7 @@ private:
     bool loadShmTexture(const QPointer<KWayland::Server::BufferInterface> &buffer);
     bool loadEglTexture(const QPointer<KWayland::Server::BufferInterface> &buffer);
     EGLImageKHR attach(const QPointer<KWayland::Server::BufferInterface> &buffer);
+    bool updateFromFBO(const QSharedPointer<QOpenGLFramebufferObject> &fbo);
     SceneOpenGL::Texture *q;
     AbstractEglBackend *m_backend;
     EGLImageKHR m_image;

scene.cpp

@@ -950,7 +950,7 @@ void WindowPixmap::create()
     if (kwinApp()->shouldUseWaylandForCompositing()) {
         // use Buffer
         updateBuffer();
-        if (m_buffer) {
+        if (m_buffer || !m_fbo.isNull()) {
             m_window->unreferencePreviousPixmap();
         }
         return;
@@ -987,7 +987,7 @@ void WindowPixmap::create()
 bool WindowPixmap::isValid() const
 {
     if (kwinApp()->shouldUseWaylandForCompositing()) {
-        return !m_buffer.isNull();
+        return !m_buffer.isNull() || !m_fbo.isNull();
     }
     return m_pixmap != XCB_PIXMAP_NONE;
 }
@@ -1004,6 +1004,12 @@ void WindowPixmap::updateBuffer()
             m_buffer = b;
             m_buffer->ref();
             QObject::connect(m_buffer.data(), &BufferInterface::aboutToBeDestroyed, m_buffer.data(), &BufferInterface::unref);
+        } else {
+            // might be an internal window
+            const auto &fbo = toplevel()->internalFramebufferObject();
+            if (!fbo.isNull()) {
+                m_fbo = fbo;
+            }
         }
     }
 }

scene.h

@@ -27,6 +27,8 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 #include <QElapsedTimer>
 
+class QOpenGLFramebufferObject;
+
 namespace KWayland
 {
 namespace Server
@@ -355,6 +357,7 @@ public:
      * @return The Wayland BufferInterface for this WindowPixmap.
      **/
     QPointer<KWayland::Server::BufferInterface> buffer() const;
+    const QSharedPointer<QOpenGLFramebufferObject> &fbo() const;
     /**
      * @brief Whether this WindowPixmap is considered as discarded. This means the window has changed in a way that a new
      * WindowPixmap should have been created already.
@@ -405,6 +408,7 @@ private:
     bool m_discarded;
     QRect m_contentsRect;
     QPointer<KWayland::Server::BufferInterface> m_buffer;
+    QSharedPointer<QOpenGLFramebufferObject> m_fbo;
 };
 
 class Scene::EffectFrame
@@ -514,6 +518,12 @@ QPointer<KWayland::Server::BufferInterface> WindowPixmap::buffer() const
     return m_buffer;
 }
 
+inline
+const QSharedPointer<QOpenGLFramebufferObject> &WindowPixmap::fbo() const
+{
+    return m_fbo;
+}
+
 template <typename T>
 inline
 T* Scene::Window::windowPixmap()

shell_client.cpp

@@ -36,6 +36,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 #include <KDesktopFile>
 
+#include <QOpenGLFramebufferObject>
 #include <QWindow>
 
 using namespace KWayland::Server;
@@ -229,6 +230,19 @@ void ShellClient::addDamage(const QRegion &damage)
     Toplevel::addDamage(damage);
 }
 
+void ShellClient::setInternalFramebufferObject(const QSharedPointer<QOpenGLFramebufferObject> &fbo)
+{
+    if (fbo.isNull()) {
+        unmap();
+        return;
+    }
+    markAsMapped();
+    m_clientSize = fbo->size();
+    setGeometry(QRect(geom.topLeft(), m_clientSize));
+    Toplevel::setInternalFramebufferObject(fbo);
+    Toplevel::addDamage(QRegion(0, 0, width(), height()));
+}
+
 void ShellClient::markAsMapped()
 {
     if (!m_unmapped) {

shell_client.h

@@ -96,6 +96,8 @@ public:
     void resizeWithChecks(int w, int h, ForceGeometry_t force = NormalGeometrySet) override;
     bool hasStrut() const override;
 
+    void setInternalFramebufferObject(const QSharedPointer<QOpenGLFramebufferObject> &fbo) override;
+
     quint32 windowId() const {
         return m_windowId;
     }

toplevel.cpp

@@ -130,6 +130,7 @@ void Toplevel::copyToDeleted(Toplevel* c)
     opaque_region = c->opaqueRegion();
     m_screen = c->m_screen;
     m_skipCloseAnimation = c->m_skipCloseAnimation;
+    m_internalFBO = c->m_internalFBO;
 }
 
 // before being deleted, remove references to everything that's now
@@ -507,6 +508,15 @@ QRegion Toplevel::inputShape() const
     }
 }
 
+void Toplevel::setInternalFramebufferObject(const QSharedPointer<QOpenGLFramebufferObject> &fbo)
+{
+    if (m_internalFBO != fbo) {
+        discardWindowPixmap();
+        m_internalFBO = fbo;
+    }
+    setDepth(32);
+}
+
 } // namespace
 
 #include "toplevel.moc"

toplevel.h

@@ -41,6 +41,8 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 // c++
 #include <functional>
 
+class QOpenGLFramebufferObject;
+
 namespace KWayland
 {
 namespace Server
@@ -356,6 +358,9 @@ public:
     KWayland::Server::SurfaceInterface *surface() const;
     void setSurface(KWayland::Server::SurfaceInterface *surface);
 
+    virtual void setInternalFramebufferObject(const QSharedPointer<QOpenGLFramebufferObject> &fbo);
+    const QSharedPointer<QOpenGLFramebufferObject> &internalFramebufferObject() const;
+
     /**
      * @brief Finds the Toplevel matching the condition expressed in @p func in @p list.
      *
@@ -495,6 +500,10 @@ private:
     bool m_skipCloseAnimation;
     quint32 m_surfaceId = 0;
     KWayland::Server::SurfaceInterface *m_surface = nullptr;
+    /**
+     * An FBO object KWin internal windows might render to.
+     **/
+    QSharedPointer<QOpenGLFramebufferObject> m_internalFBO;
     // when adding new data members, check also copyToDeleted()
 };
 
@@ -731,6 +740,11 @@ inline KWayland::Server::SurfaceInterface *Toplevel::surface() const
     return m_surface;
 }
 
+inline const QSharedPointer<QOpenGLFramebufferObject> &Toplevel::internalFramebufferObject() const
+{
+    return m_internalFBO;
+}
+
 template <class T, class U>
 inline T *Toplevel::findInList(const QList<T*> &list, std::function<bool (const U*)> func)
 {