From 916f1f4ea8d49d857fd15681efec18ba06f1027b Mon Sep 17 00:00:00 2001
From: Xaver Hugl <xaver.hugl@gmail.com>
Date: Wed, 13 Apr 2022 21:39:46 +0200
Subject: [PATCH] backends/drm: prune format modifiers when tests fail

When explicit modifiers are used, it can happen that Mesa chooses modifiers that make
the display hardware hit bandwidth limits. In that case, atomic tests fail and the
outputs don't work, or KWin may even crash.
In order to work around that, DrmGpu now removes the used modifier whenever an atomic
test fails, and tries to find a working combination of outputs and modifiers.
---
 src/backends/drm/drm_buffer_gbm.cpp |  2 +-
 src/backends/drm/drm_gpu.cpp        | 16 +++++++++------
 src/backends/drm/drm_pipeline.cpp   | 32 +++++++++++++++++++----------
 src/backends/drm/drm_pipeline.h     |  2 ++
 4 files changed, 34 insertions(+), 18 deletions(-)

diff --git a/src/backends/drm/drm_buffer_gbm.cpp b/src/backends/drm/drm_buffer_gbm.cpp
index 5c56190dfd..8b2ce6d83f 100644
--- a/src/backends/drm/drm_buffer_gbm.cpp
+++ b/src/backends/drm/drm_buffer_gbm.cpp
@@ -144,7 +144,7 @@ QSharedPointer<GLTexture> GbmBuffer::createTexture(EGLDisplay eglDisplay) const
 }
 
 DrmGbmBuffer::DrmGbmBuffer(DrmGpu *gpu, GbmSurface *surface, gbm_bo *bo)
-    : DrmBuffer(gpu, gbm_bo_get_format(bo), gbm_bo_get_modifier(bo))
+    : DrmBuffer(gpu, gbm_bo_get_format(bo), surface && surface->modifiers().isEmpty() ? DRM_FORMAT_MOD_INVALID : gbm_bo_get_modifier(bo))
     , GbmBuffer(surface, bo)
 {
     initialize();
diff --git a/src/backends/drm/drm_gpu.cpp b/src/backends/drm/drm_gpu.cpp
index 8b72ba6458..5fb8256fc4 100644
--- a/src/backends/drm/drm_gpu.cpp
+++ b/src/backends/drm/drm_gpu.cpp
@@ -376,9 +376,11 @@ bool DrmGpu::checkCrtcAssignment(QVector<DrmConnector *> connectors, const QVect
             auto crtcsLeft = crtcs;
             crtcsLeft.removeOne(currentCrtc);
             pipeline->setCrtc(currentCrtc);
-            if (checkCrtcAssignment(connectors, crtcsLeft)) {
-                return true;
-            }
+            do {
+                if (checkCrtcAssignment(connectors, crtcsLeft)) {
+                    return true;
+                }
+            } while (pipeline->pruneModifier());
         }
     }
     for (const auto &crtc : qAsConst(crtcs)) {
@@ -386,9 +388,11 @@ bool DrmGpu::checkCrtcAssignment(QVector<DrmConnector *> connectors, const QVect
             auto crtcsLeft = crtcs;
             crtcsLeft.removeOne(crtc);
             pipeline->setCrtc(crtc);
-            if (checkCrtcAssignment(connectors, crtcsLeft)) {
-                return true;
-            }
+            do {
+                if (checkCrtcAssignment(connectors, crtcsLeft)) {
+                    return true;
+                }
+            } while (pipeline->pruneModifier());
         }
     }
     return false;
diff --git a/src/backends/drm/drm_pipeline.cpp b/src/backends/drm/drm_pipeline.cpp
index 6cdfd29423..8d3bc34200 100644
--- a/src/backends/drm/drm_pipeline.cpp
+++ b/src/backends/drm/drm_pipeline.cpp
@@ -409,20 +409,23 @@ DrmOutput *DrmPipeline::output() const
     return m_output;
 }
 
-static const QMap<uint32_t, QVector<uint64_t>> legacyFormats = {
-    {DRM_FORMAT_XRGB8888, {}}};
-
 QMap<uint32_t, QVector<uint64_t>> DrmPipeline::formats() const
 {
-    if (m_pending.crtc) {
-        if (m_pending.crtc->primaryPlane()) {
-            return m_pending.crtc->primaryPlane()->formats();
-        } else {
-            return legacyFormats;
-        }
-    } else {
-        return {};
+    return m_pending.formats;
+}
+
+bool DrmPipeline::pruneModifier()
+{
+    if (m_pending.layer->currentBuffer()->modifier() == DRM_FORMAT_MOD_NONE
+        || m_pending.layer->currentBuffer()->modifier() == DRM_FORMAT_MOD_INVALID) {
+        return false;
     }
+    auto &modifiers = m_pending.formats[m_pending.layer->currentBuffer()->format()];
+    if (modifiers.count() <= 1) {
+        return false;
+    }
+    modifiers.removeOne(m_pending.layer->currentBuffer()->modifier());
+    return true;
 }
 
 bool DrmPipeline::needsModeset() const
@@ -608,9 +611,16 @@ Output::RgbRange DrmPipeline::rgbRange() const
     return m_pending.rgbRange;
 }
 
+static const QMap<uint32_t, QVector<uint64_t>> legacyFormats = {{DRM_FORMAT_XRGB8888, {}}};
+
 void DrmPipeline::setCrtc(DrmCrtc *crtc)
 {
     m_pending.crtc = crtc;
+    if (crtc) {
+        m_pending.formats = crtc->primaryPlane() ? crtc->primaryPlane()->formats() : legacyFormats;
+    } else {
+        m_pending.formats = {};
+    }
 }
 
 void DrmPipeline::setMode(const QSharedPointer<DrmConnectorMode> &mode)
diff --git a/src/backends/drm/drm_pipeline.h b/src/backends/drm/drm_pipeline.h
index d88c19b211..a2d27f9763 100644
--- a/src/backends/drm/drm_pipeline.h
+++ b/src/backends/drm/drm_pipeline.h
@@ -85,6 +85,7 @@ public:
     QSize bufferSize() const;
 
     QMap<uint32_t, QVector<uint64_t>> formats() const;
+    bool pruneModifier();
 
     void setOutput(DrmOutput *output);
     DrmOutput *output() const;
@@ -158,6 +159,7 @@ private:
     struct State
     {
         DrmCrtc *crtc = nullptr;
+        QMap<uint32_t, QVector<uint64_t>> formats;
         bool active = true; // whether or not the pipeline should be currently used
         bool enabled = true; // whether or not the pipeline needs a crtc
         QSharedPointer<DrmConnectorMode> mode;