backends/drm: prune format modifiers when tests fail

When explicit modifiers are used, it can happen that Mesa chooses modifiers that make
the display hardware hit bandwidth limits. In that case, atomic tests fail and the
outputs don't work, or KWin may even crash.
In order to work around that, DrmGpu now removes the used modifier whenever an atomic
test fails, and tries to find a working combination of outputs and modifiers.

src/backends/drm/drm_buffer_gbm.cpp

@@ -144,7 +144,7 @@ QSharedPointer<GLTexture> GbmBuffer::createTexture(EGLDisplay eglDisplay) const
 }
 
 DrmGbmBuffer::DrmGbmBuffer(DrmGpu *gpu, GbmSurface *surface, gbm_bo *bo)
-    : DrmBuffer(gpu, gbm_bo_get_format(bo), gbm_bo_get_modifier(bo))
+    : DrmBuffer(gpu, gbm_bo_get_format(bo), surface && surface->modifiers().isEmpty() ? DRM_FORMAT_MOD_INVALID : gbm_bo_get_modifier(bo))
     , GbmBuffer(surface, bo)
 {
     initialize();

src/backends/drm/drm_gpu.cpp

@@ -376,9 +376,11 @@ bool DrmGpu::checkCrtcAssignment(QVector<DrmConnector *> connectors, const QVect
             auto crtcsLeft = crtcs;
             crtcsLeft.removeOne(currentCrtc);
             pipeline->setCrtc(currentCrtc);
-            if (checkCrtcAssignment(connectors, crtcsLeft)) {
-                return true;
-            }
+            do {
+                if (checkCrtcAssignment(connectors, crtcsLeft)) {
+                    return true;
+                }
+            } while (pipeline->pruneModifier());
         }
     }
     for (const auto &crtc : qAsConst(crtcs)) {
@@ -386,9 +388,11 @@ bool DrmGpu::checkCrtcAssignment(QVector<DrmConnector *> connectors, const QVect
             auto crtcsLeft = crtcs;
             crtcsLeft.removeOne(crtc);
             pipeline->setCrtc(crtc);
-            if (checkCrtcAssignment(connectors, crtcsLeft)) {
-                return true;
-            }
+            do {
+                if (checkCrtcAssignment(connectors, crtcsLeft)) {
+                    return true;
+                }
+            } while (pipeline->pruneModifier());
         }
     }
     return false;

src/backends/drm/drm_pipeline.cpp

@@ -409,20 +409,23 @@ DrmOutput *DrmPipeline::output() const
     return m_output;
 }
 
-static const QMap<uint32_t, QVector<uint64_t>> legacyFormats = {
-    {DRM_FORMAT_XRGB8888, {}}};
-
 QMap<uint32_t, QVector<uint64_t>> DrmPipeline::formats() const
 {
-    if (m_pending.crtc) {
-        if (m_pending.crtc->primaryPlane()) {
-            return m_pending.crtc->primaryPlane()->formats();
-        } else {
-            return legacyFormats;
-        }
-    } else {
-        return {};
+    return m_pending.formats;
+}
+
+bool DrmPipeline::pruneModifier()
+{
+    if (m_pending.layer->currentBuffer()->modifier() == DRM_FORMAT_MOD_NONE
+        || m_pending.layer->currentBuffer()->modifier() == DRM_FORMAT_MOD_INVALID) {
+        return false;
     }
+    auto &modifiers = m_pending.formats[m_pending.layer->currentBuffer()->format()];
+    if (modifiers.count() <= 1) {
+        return false;
+    }
+    modifiers.removeOne(m_pending.layer->currentBuffer()->modifier());
+    return true;
 }
 
 bool DrmPipeline::needsModeset() const
@@ -608,9 +611,16 @@ Output::RgbRange DrmPipeline::rgbRange() const
     return m_pending.rgbRange;
 }
 
+static const QMap<uint32_t, QVector<uint64_t>> legacyFormats = {{DRM_FORMAT_XRGB8888, {}}};
+
 void DrmPipeline::setCrtc(DrmCrtc *crtc)
 {
     m_pending.crtc = crtc;
+    if (crtc) {
+        m_pending.formats = crtc->primaryPlane() ? crtc->primaryPlane()->formats() : legacyFormats;
+    } else {
+        m_pending.formats = {};
+    }
 }
 
 void DrmPipeline::setMode(const QSharedPointer<DrmConnectorMode> &mode)

src/backends/drm/drm_pipeline.h

@@ -85,6 +85,7 @@ public:
     QSize bufferSize() const;
 
     QMap<uint32_t, QVector<uint64_t>> formats() const;
+    bool pruneModifier();
 
     void setOutput(DrmOutput *output);
     DrmOutput *output() const;
@@ -158,6 +159,7 @@ private:
     struct State
     {
         DrmCrtc *crtc = nullptr;
+        QMap<uint32_t, QVector<uint64_t>> formats;
         bool active = true; // whether or not the pipeline should be currently used
         bool enabled = true; // whether or not the pipeline needs a crtc
         QSharedPointer<DrmConnectorMode> mode;