screencasting: on memfd, skip the QImage step

We were using QImage as an intermediary step. GL -> QImage -> spa
buffer. While it abstracted things out neatly and eventually helped with
debugging, it was unnecessary and woudl present some handicaps, such as
the lack of a QImage::Format_BGRA.
So we just it out to download right into the buffer.

BUG: 466655
(cherry picked from commit 121454580711c409b612d06865ab9d221dcbac6b)

src/plugins/screencast/outputscreencastsource.cpp

@@ -44,11 +44,11 @@ QSize OutputScreenCastSource::textureSize() const
     return m_output->pixelSize();
 }
 
-void OutputScreenCastSource::render(QImage *image)
+void OutputScreenCastSource::render(spa_data *spa, spa_video_format format)
 {
     const std::shared_ptr<GLTexture> outputTexture = Compositor::self()->scene()->textureForOutput(m_output);
     if (outputTexture) {
-        grabTexture(outputTexture.get(), image);
+        grabTexture(outputTexture.get(), spa, format);
     }
 }
 

src/plugins/screencast/outputscreencastsource.h

@@ -27,7 +27,7 @@ public:
     quint32 drmFormat() const override;
 
     void render(GLFramebuffer *target) override;
-    void render(QImage *image) override;
+    void render(spa_data *spa, spa_video_format format) override;
     std::chrono::nanoseconds clock() const override;
 
 private:

src/plugins/screencast/regionscreencastsource.cpp

@@ -110,10 +110,10 @@ void RegionScreenCastSource::render(GLFramebuffer *target)
     GLFramebuffer::popFramebuffer();
 }
 
-void RegionScreenCastSource::render(QImage *image)
+void RegionScreenCastSource::render(spa_data *spa, spa_video_format format)
 {
     ensureTexture();
-    grabTexture(m_renderedTexture.get(), image);
+    grabTexture(m_renderedTexture.get(), spa, format);
 }
 
 }

src/plugins/screencast/regionscreencastsource.h

@@ -28,7 +28,7 @@ public:
     QSize textureSize() const override;
 
     void render(GLFramebuffer *target) override;
-    void render(QImage *image) override;
+    void render(spa_data *spa, spa_video_format format) override;
     std::chrono::nanoseconds clock() const override;
 
     QRect region() const

src/plugins/screencast/screencastsource.h

@@ -7,6 +7,8 @@
 #pragma once
 
 #include <QObject>
+#include <spa/buffer/buffer.h>
+#include <spa/param/video/raw.h>
 
 namespace KWin
 {
@@ -25,7 +27,7 @@ public:
     virtual QSize textureSize() const = 0;
 
     virtual void render(GLFramebuffer *target) = 0;
-    virtual void render(QImage *image) = 0;
+    virtual void render(spa_data *spa, spa_video_format format) = 0;
     virtual std::chrono::nanoseconds clock() const = 0;
 
 Q_SIGNALS:

src/plugins/screencast/screencaststream.cpp

@@ -450,20 +450,20 @@ void ScreenCastStream::recordFrame(const QRegion &_damagedRegion)
         const int bpp = data && !hasAlpha ? 3 : 4;
         const uint stride = SPA_ROUND_UP_N(size.width() * bpp, 4);
 
-        QImage dest(data, size.width(), size.height(), stride, hasAlpha ? QImage::Format_RGBA8888_Premultiplied : QImage::Format_RGB888);
-        if (dest.sizeInBytes() > spa_data->maxsize) {
+        if ((stride * size.height()) > spa_data->maxsize) {
             qCDebug(KWIN_SCREENCAST) << "Failed to record frame: frame is too big";
             pw_stream_queue_buffer(pwStream, buffer);
             return;
         }
 
-        spa_data->chunk->size = dest.sizeInBytes();
-        spa_data->chunk->stride = dest.bytesPerLine();
+        spa_data->chunk->stride = stride;
+        spa_data->chunk->size = stride * size.height();
 
-        m_source->render(&dest);
+        m_source->render(spa_data, videoFormat.format);
 
         auto cursor = Cursors::self()->currentCursor();
         if (m_cursor.mode == KWaylandServer::ScreencastV1Interface::Embedded && exclusiveContains(m_cursor.viewport, cursor->pos())) {
+            QImage dest(data, size.width(), size.height(), stride, hasAlpha ? QImage::Format_RGBA8888_Premultiplied : QImage::Format_RGB888);
             QPainter painter(&dest);
             const auto position = (cursor->pos() - m_cursor.viewport.topLeft() - cursor->hotspot()) * m_cursor.scale;
             painter.drawImage(QRect{position.toPoint(), cursor->image().size()}, cursor->image());

src/plugins/screencast/screencastutils.h

@@ -8,6 +8,8 @@
 
 #include "kwinglplatform.h"
 #include "kwingltexture.h"
+#include <spa/buffer/buffer.h>
+#include <spa/param/video/raw.h>
 
 namespace KWin
 {
@@ -25,28 +27,29 @@ static void mirrorVertically(uchar *data, int height, int stride)
     }
 }
 
-static GLenum closestGLType(const QImage &image)
+static GLenum closestGLType(spa_video_format format)
 {
-    switch (image.format()) {
-    case QImage::Format_RGB888:
+    switch (format) {
+    case SPA_VIDEO_FORMAT_RGB:
         return GL_RGB;
-    case QImage::Format_BGR888:
+    case SPA_VIDEO_FORMAT_BGR:
         return GL_BGR;
-    case QImage::Format_RGB32:
-    case QImage::Format_RGBX8888:
-    case QImage::Format_RGBA8888:
-    case QImage::Format_RGBA8888_Premultiplied:
+    case SPA_VIDEO_FORMAT_RGBx:
+    case SPA_VIDEO_FORMAT_RGBA:
         return GL_RGBA;
+    case SPA_VIDEO_FORMAT_BGRA:
+    case SPA_VIDEO_FORMAT_BGRx:
+        return GL_BGRA;
     default:
-        qDebug() << "unknown format" << image.format();
+        qDebug() << "unknown format" << format;
         return GL_RGBA;
     }
 }
 
-static void grabTexture(GLTexture *texture, QImage *image)
+static void grabTexture(GLTexture *texture, spa_data *spa, spa_video_format format)
 {
     const bool invert = !texture->isYInverted();
-    Q_ASSERT(texture->size() == image->size());
+    const QSize size = texture->size();
     bool isGLES = GLPlatform::instance()->isGLES();
     bool invertNeeded = isGLES ^ invert;
     const bool invertNeededAndSupported = invertNeeded && GLPlatform::instance()->supports(PackInvert);
@@ -58,11 +61,11 @@ static void grabTexture(GLTexture *texture, QImage *image)
 
     texture->bind();
     if (GLPlatform::instance()->isGLES()) {
-        glReadPixels(0, 0, image->width(), image->height(), closestGLType(*image), GL_UNSIGNED_BYTE, (GLvoid *)image->bits());
+        glReadPixels(0, 0, size.width(), size.height(), closestGLType(format), GL_UNSIGNED_BYTE, spa->data);
     } else if (GLPlatform::instance()->glVersion() >= kVersionNumber(4, 5)) {
-        glGetTextureImage(texture->texture(), 0, closestGLType(*image), GL_UNSIGNED_BYTE, image->sizeInBytes(), image->bits());
+        glGetTextureImage(texture->texture(), 0, closestGLType(format), GL_UNSIGNED_BYTE, spa->chunk->size, spa->data);
     } else {
-        glGetTexImage(texture->target(), 0, closestGLType(*image), GL_UNSIGNED_BYTE, image->bits());
+        glGetTexImage(texture->target(), 0, closestGLType(format), GL_UNSIGNED_BYTE, spa->data);
     }
 
     if (invertNeededAndSupported) {
@@ -70,7 +73,7 @@ static void grabTexture(GLTexture *texture, QImage *image)
             glPixelStorei(GL_PACK_INVERT_MESA, prev);
         }
     } else if (invertNeeded) {
-        mirrorVertically(image->bits(), image->height(), image->bytesPerLine());
+        mirrorVertically(static_cast<uchar *>(spa->data), size.height(), spa->chunk->stride);
     }
 }
 

src/plugins/screencast/windowscreencastsource.cpp

@@ -46,13 +46,13 @@ QSize WindowScreenCastSource::textureSize() const
     return m_window->clientGeometry().size().toSize();
 }
 
-void WindowScreenCastSource::render(QImage *image)
+void WindowScreenCastSource::render(spa_data *spa, spa_video_format format)
 {
     GLTexture offscreenTexture(hasAlphaChannel() ? GL_RGBA8 : GL_RGB8, textureSize());
     GLFramebuffer offscreenTarget(&offscreenTexture);
 
     render(&offscreenTarget);
-    grabTexture(&offscreenTexture, image);
+    grabTexture(&offscreenTexture, spa, format);
 }
 
 void WindowScreenCastSource::render(GLFramebuffer *target)

src/plugins/screencast/windowscreencastsource.h

@@ -27,7 +27,7 @@ public:
     QSize textureSize() const override;
 
     void render(GLFramebuffer *target) override;
-    void render(QImage *image) override;
+    void render(spa_data *spa, spa_video_format format) override;
     std::chrono::nanoseconds clock() const override;
 
 private: