dmabuf feedback: limit modifiers to those supported by egl

src/backends/drm/egl_gbm_backend.cpp

@@ -27,6 +27,7 @@
 #include "shadowbuffer.h"
 #include "drm_pipeline.h"
 #include "drm_abstract_output.h"
+#include "egl_dmabuf.h"
 // kwin libs
 #include <kwinglplatform.h>
 #include <kwineglimagetexture.h>
@@ -659,9 +660,12 @@ bool EglGbmBackend::scanout(AbstractOutput *drmOutput, SurfaceItem *surfaceItem)
             // atm no format changes are sent as those might require a modeset
             // and thus require more elaborate feedback
             const auto &mods = drmOutput->pipeline()->supportedModifiers(output.current.format.drmFormat);
+            const auto &supportedModifiers = primaryBackend()->dmabuf()->supportedFormats()[output.current.format.drmFormat];
             for (const auto &mod : mods) {
+                if (supportedModifiers.contains(mod)) {
                     tranche.formatTable[output.current.format.drmFormat] << mod;
                 }
+            }
             if (tranche.formatTable.isEmpty()) {
                 output.scanoutCandidate->dmabufFeedbackV1()->setTranches({});
             } else {

src/platformsupport/scenes/opengl/abstract_egl_backend.cpp

@@ -398,4 +398,9 @@ bool AbstractEglBackend::prefer10bpc() const
     return false;
 }
 
+EglDmabuf *AbstractEglBackend::dmabuf() const
+{
+    return m_dmaBuf;
+}
+
 }

src/platformsupport/scenes/opengl/abstract_egl_backend.h

@@ -72,6 +72,7 @@ public:
 
     dev_t deviceId() const;
     virtual bool prefer10bpc() const;
+    EglDmabuf *dmabuf() const;
 
 protected:
     AbstractEglBackend(dev_t deviceId = 0);

src/platformsupport/scenes/opengl/egl_dmabuf.cpp

@@ -440,12 +440,7 @@ void EglDmabuf::setSupportedFormatsAndModifiers()
 
     filterFormatsWithMultiplePlanes(formats);
 
-    auto queryFormats = [&formats, &eglDisplay](int bpc) {
-        QHash<uint32_t, QSet<uint64_t>> set;
     for (auto format : qAsConst(formats)) {
-            if (bpc != bpcForFormat(format)) {
-                continue;
-            }
         if (eglQueryDmaBufModifiersEXT != nullptr) {
             EGLint count = 0;
             const EGLBoolean success = eglQueryDmaBufModifiersEXT(eglDisplay, format, 0, nullptr, nullptr, &count);
@@ -456,12 +451,20 @@ void EglDmabuf::setSupportedFormatsAndModifiers()
                     for (const uint64_t &mod : qAsConst(modifiers)) {
                         modifiersSet.insert(mod);
                     }
-                        set.insert(format, modifiersSet);
+                    m_supportedFormats.insert(format, modifiersSet);
                     continue;
                 }
             }
         }
-            set.insert(format, QSet<uint64_t>());
+        m_supportedFormats.insert(format, QSet<uint64_t>());
+    }
+
+    auto filterFormats = [this](int bpc) {
+        QHash<uint32_t, QSet<uint64_t>> set;
+        for (auto it = m_supportedFormats.constBegin(); it != m_supportedFormats.constEnd(); it++) {
+            if (bpcForFormat(it.key()) == bpc) {
+                set.insert(it.key(), it.value());
+            }
         }
         return set;
     };
@@ -470,18 +473,18 @@ void EglDmabuf::setSupportedFormatsAndModifiers()
         tranches.append({
             .device = m_backend->deviceId(),
             .flags = {},
-            .formatTable = queryFormats(10),
+            .formatTable = filterFormats(10),
         });
     }
     tranches.append({
         .device = m_backend->deviceId(),
         .flags = {},
-        .formatTable = queryFormats(8),
+        .formatTable = filterFormats(8),
     });
     tranches.append({
         .device = m_backend->deviceId(),
         .flags = {},
-        .formatTable = queryFormats(-1),
+        .formatTable = filterFormats(-1),
     });
     LinuxDmaBufV1RendererInterface::setSupportedFormatsAndModifiers(tranches);
 }

src/platformsupport/scenes/opengl/egl_dmabuf.h

@@ -67,6 +67,10 @@ public:
                                                             const QSize &size,
                                                             quint32 flags) override;
 
+    QHash<uint32_t, QSet<uint64_t>> supportedFormats() const {
+        return m_supportedFormats;
+    }
+
 private:
     EGLImage createImage(const QVector<KWaylandServer::LinuxDmaBufV1Plane> &planes,
                          uint32_t format,
@@ -80,6 +84,7 @@ private:
     void setSupportedFormatsAndModifiers();
 
     AbstractEglBackend *m_backend;
+    QHash<uint32_t, QSet<uint64_t>> m_supportedFormats;
 
     friend class EglDmabufBuffer;
 };