backends/drm: improve dmabuf feedback

Instead of only allowing the current format, send the default tranches modified
to only contain formats and modifiers suitable for scanout.
In order to not fail when we can't do direct scanout with a given format
(because that may require a modeset, which we don't allow), keep a blacklist
of attempted formats and modifiers for the current client.

src/backends/drm/drm_pipeline.cpp

@@ -445,13 +445,18 @@ DrmOutput *DrmPipeline::output() const
     return m_output;
 }
 
+static const QMap<uint32_t, QVector<uint64_t>> legacyFormats = {
+    {DRM_FORMAT_XRGB8888, {}},
+    {DRM_FORMAT_ARGB8888, {}}
+};
+
 bool DrmPipeline::isFormatSupported(uint32_t drmFormat) const
 {
     if (pending.crtc) {
         if (pending.crtc->primaryPlane()) {
             return pending.crtc->primaryPlane()->formats().contains(drmFormat);
         } else {
-            return drmFormat == DRM_FORMAT_XRGB8888 || drmFormat == DRM_FORMAT_ARGB8888;
+            return legacyFormats.keys().contains(drmFormat);
         }
     } else {
         return false;
@@ -467,6 +472,19 @@ QVector<uint64_t> DrmPipeline::supportedModifiers(uint32_t drmFormat) const
     }
 }
 
+QMap<uint32_t, QVector<uint64_t>> DrmPipeline::supportedFormats() const
+{
+    if (pending.crtc) {
+        if (pending.crtc->primaryPlane()) {
+            return pending.crtc->primaryPlane()->formats();
+        } else {
+            return legacyFormats;
+        }
+    } else {
+        return {};
+    }
+}
+
 bool DrmPipeline::needsModeset() const
 {
     return pending.crtc != m_current.crtc

src/backends/drm/drm_pipeline.h

@@ -81,6 +81,7 @@ public:
 
     bool isFormatSupported(uint32_t drmFormat) const;
     QVector<uint64_t> supportedModifiers(uint32_t drmFormat) const;
+    QMap<uint32_t, QVector<uint64_t>> supportedFormats() const;
 
     void setOutput(DrmOutput *output);
     DrmOutput *output() const;

src/backends/drm/egl_gbm_backend.cpp

@@ -529,9 +529,9 @@ QRegion EglGbmBackend::beginFrame(AbstractOutput *drmOutput)
         qCDebug(KWIN_DRM) << "Direct scanout stopped on output" << output.output->name();
     }
     output.scanoutSurface = nullptr;
-    if (output.scanoutCandidate) {
+    if (output.scanoutCandidate.surface) {
-        output.oldScanoutCandidate = output.scanoutCandidate;
+        output.oldScanoutCandidate = output.scanoutCandidate.surface;
-        output.scanoutCandidate = nullptr;
+        output.scanoutCandidate = {};
     } else if (output.oldScanoutCandidate && output.oldScanoutCandidate->dmabufFeedbackV1()) {
         output.oldScanoutCandidate->dmabufFeedbackV1()->setTranches({});
     }
@@ -642,7 +642,8 @@ bool EglGbmBackend::scanout(AbstractOutput *drmOutput, SurfaceItem *surfaceItem)
         return false;
     }
     Output &output = m_outputs[drmOutput];
-    if (buffer->size() != output.output->modeSize()) {
+    const auto planes = buffer->planes();
+    if (buffer->size() != output.output->modeSize() || planes.isEmpty()) {
         return false;
     }
     if (output.oldScanoutCandidate && output.oldScanoutCandidate != surface) {
@@ -651,35 +652,45 @@ bool EglGbmBackend::scanout(AbstractOutput *drmOutput, SurfaceItem *surfaceItem)
         }
         output.oldScanoutCandidate = nullptr;
     }
-    output.scanoutCandidate = surface;
+    if (output.scanoutCandidate.surface != surface) {
-    const auto &sendFeedback = [&output, this]() {
+        output.scanoutCandidate.attemptedFormats = {};
-        if (const auto &drmOutput = qobject_cast<DrmOutput *>(output.output); drmOutput && output.scanoutCandidate->dmabufFeedbackV1()) {
+    }
-            KWaylandServer::LinuxDmaBufV1Feedback::Tranche tranche;
+    output.scanoutCandidate.surface = surface;
-            tranche.device = m_gpu->deviceId();
+    const auto &sendFeedback = [&output, &buffer, &planes, this]() {
-            tranche.flags = KWaylandServer::LinuxDmaBufV1Feedback::TrancheFlag::Scanout;
+        if (!output.scanoutCandidate.attemptedFormats[buffer->format()].contains(planes.first().modifier)) {
-            // atm no format changes are sent as those might require a modeset
+            output.scanoutCandidate.attemptedFormats[buffer->format()] << planes.first().modifier;
-            // and thus require more elaborate feedback
+        }
-            const auto &mods = drmOutput->pipeline()->supportedModifiers(output.current.format.drmFormat);
+        if (const auto &drmOutput = qobject_cast<DrmOutput *>(output.output); drmOutput && output.scanoutCandidate.surface->dmabufFeedbackV1()) {
-            const auto &supportedModifiers = primaryBackend()->dmabuf()->supportedFormats()[output.current.format.drmFormat];
+            QVector<KWaylandServer::LinuxDmaBufV1Feedback::Tranche> scanoutTranches;
-            for (const auto &mod : mods) {
+            const auto &drmFormats = drmOutput->pipeline()->supportedFormats();
-                if (supportedModifiers.contains(mod)) {
+            const auto tranches = primaryBackend()->dmabuf()->tranches();
-                    tranche.formatTable[output.current.format.drmFormat] << mod;
+            for (const auto &tranche : tranches) {
+                KWaylandServer::LinuxDmaBufV1Feedback::Tranche scanoutTranche;
+                for (auto it = tranche.formatTable.constBegin(); it != tranche.formatTable.constEnd(); it++) {
+                    const uint32_t format = it.key();
+                    const auto trancheModifiers = it.value();
+                    const auto drmModifiers = drmFormats[format];
+                    for (const auto &mod : trancheModifiers) {
+                        if (drmModifiers.contains(mod) && !output.scanoutCandidate.attemptedFormats[format].contains(mod)) {
+                            scanoutTranche.formatTable[format] << mod;
+                        }
+                    }
+                }
+                if (!scanoutTranche.formatTable.isEmpty()) {
+                    scanoutTranche.device = m_gpu->deviceId();
+                    scanoutTranche.flags = KWaylandServer::LinuxDmaBufV1Feedback::TrancheFlag::Scanout;
+                    scanoutTranches << scanoutTranche;
                 }
             }
-            if (tranche.formatTable.isEmpty()) {
+            output.scanoutCandidate.surface->dmabufFeedbackV1()->setTranches(scanoutTranches);
-                output.scanoutCandidate->dmabufFeedbackV1()->setTranches({});
-            } else {
-                output.scanoutCandidate->dmabufFeedbackV1()->setTranches({tranche});
-            }
         }
     };
-    if (!buffer->planes().count() || !output.output->isFormatSupported(buffer->format())) {
+    if (!output.output->isFormatSupported(buffer->format())) {
         sendFeedback();
         return false;
     }
 
     gbm_bo *importedBuffer;
-    const auto planes = buffer->planes();
     if (planes.first().modifier != DRM_FORMAT_MOD_INVALID
         || planes.first().offset > 0
         || planes.count() > 1) {
@@ -740,7 +751,10 @@ bool EglGbmBackend::scanout(AbstractOutput *drmOutput, SurfaceItem *surfaceItem)
         output.scanoutSurface = surface;
         return true;
     } else {
-        sendFeedback();
+        // TODO clean the modeset and direct scanout code paths up
+        if (!m_gpu->needsModeset()) {
+            sendFeedback();
+        }
         return false;
     }
 }

src/backends/drm/egl_gbm_backend.h

@@ -108,7 +108,10 @@ private:
         } old, current;
 
         KWaylandServer::SurfaceInterface *scanoutSurface = nullptr;
-        QPointer<KWaylandServer::SurfaceInterface> scanoutCandidate;
+        struct {
+            QPointer<KWaylandServer::SurfaceInterface> surface;
+            QMap<uint32_t, QVector<uint64_t>> attemptedFormats;
+        } scanoutCandidate;
         QPointer<KWaylandServer::SurfaceInterface> oldScanoutCandidate;
     };
 

src/platformsupport/scenes/opengl/egl_dmabuf.cpp

@@ -440,6 +440,7 @@ void EglDmabuf::setSupportedFormatsAndModifiers()
 
     filterFormatsWithMultiplePlanes(formats);
 
+    QHash<uint32_t, QSet<uint64_t>> supportedFormats;
     for (auto format : qAsConst(formats)) {
         if (eglQueryDmaBufModifiersEXT != nullptr) {
             EGLint count = 0;
@@ -451,42 +452,41 @@ void EglDmabuf::setSupportedFormatsAndModifiers()
                     for (const uint64_t &mod : qAsConst(modifiers)) {
                         modifiersSet.insert(mod);
                     }
-                    m_supportedFormats.insert(format, modifiersSet);
+                    supportedFormats.insert(format, modifiersSet);
                     continue;
                 }
             }
         }
-        m_supportedFormats.insert(format, QSet<uint64_t>());
+        supportedFormats.insert(format, QSet<uint64_t>());
     }
 
-    auto filterFormats = [this](int bpc) {
+    auto filterFormats = [&supportedFormats](int bpc) {
         QHash<uint32_t, QSet<uint64_t>> set;
-        for (auto it = m_supportedFormats.constBegin(); it != m_supportedFormats.constEnd(); it++) {
+        for (auto it = supportedFormats.constBegin(); it != supportedFormats.constEnd(); it++) {
             if (bpcForFormat(it.key()) == bpc) {
                 set.insert(it.key(), it.value());
             }
         }
         return set;
     };
-    QVector<KWaylandServer::LinuxDmaBufV1Feedback::Tranche> tranches;
     if (m_backend->prefer10bpc()) {
-        tranches.append({
+        m_tranches.append({
             .device = m_backend->deviceId(),
             .flags = {},
             .formatTable = filterFormats(10),
         });
     }
-    tranches.append({
+    m_tranches.append({
         .device = m_backend->deviceId(),
         .flags = {},
         .formatTable = filterFormats(8),
     });
-    tranches.append({
+    m_tranches.append({
         .device = m_backend->deviceId(),
         .flags = {},
         .formatTable = filterFormats(-1),
     });
-    LinuxDmaBufV1RendererInterface::setSupportedFormatsAndModifiers(tranches);
+    LinuxDmaBufV1RendererInterface::setSupportedFormatsAndModifiers(m_tranches);
 }
 
 }

src/platformsupport/scenes/opengl/egl_dmabuf.h

@@ -67,8 +67,8 @@ public:
                                                             const QSize &size,
                                                             quint32 flags) override;
 
-    QHash<uint32_t, QSet<uint64_t>> supportedFormats() const {
+    QVector<KWaylandServer::LinuxDmaBufV1Feedback::Tranche> tranches() const {
-        return m_supportedFormats;
+        return m_tranches;
     }
 
 private:
@@ -84,7 +84,7 @@ private:
     void setSupportedFormatsAndModifiers();
 
     AbstractEglBackend *m_backend;
-    QHash<uint32_t, QSet<uint64_t>> m_supportedFormats;
+    QVector<KWaylandServer::LinuxDmaBufV1Feedback::Tranche> m_tranches;
 
     friend class EglDmabufBuffer;
 };