diff --git a/src/backends/drm/drm_buffer_gbm.cpp b/src/backends/drm/drm_buffer_gbm.cpp
index 5c56190dfd..8b2ce6d83f 100644
--- a/src/backends/drm/drm_buffer_gbm.cpp
+++ b/src/backends/drm/drm_buffer_gbm.cpp
@@ -144,7 +144,7 @@ QSharedPointer<GLTexture> GbmBuffer::createTexture(EGLDisplay eglDisplay) const
 }
 
 DrmGbmBuffer::DrmGbmBuffer(DrmGpu *gpu, GbmSurface *surface, gbm_bo *bo)
-    : DrmBuffer(gpu, gbm_bo_get_format(bo), gbm_bo_get_modifier(bo))
+    : DrmBuffer(gpu, gbm_bo_get_format(bo), surface && surface->modifiers().isEmpty() ? DRM_FORMAT_MOD_INVALID : gbm_bo_get_modifier(bo))
     , GbmBuffer(surface, bo)
 {
     initialize();
diff --git a/src/backends/drm/drm_gpu.cpp b/src/backends/drm/drm_gpu.cpp
index 8b72ba6458..5fb8256fc4 100644
--- a/src/backends/drm/drm_gpu.cpp
+++ b/src/backends/drm/drm_gpu.cpp
@@ -376,9 +376,11 @@ bool DrmGpu::checkCrtcAssignment(QVector<DrmConnector *> connectors, const QVect
             auto crtcsLeft = crtcs;
             crtcsLeft.removeOne(currentCrtc);
             pipeline->setCrtc(currentCrtc);
-            if (checkCrtcAssignment(connectors, crtcsLeft)) {
-                return true;
-            }
+            do {
+                if (checkCrtcAssignment(connectors, crtcsLeft)) {
+                    return true;
+                }
+            } while (pipeline->pruneModifier());
         }
     }
     for (const auto &crtc : qAsConst(crtcs)) {
@@ -386,9 +388,11 @@ bool DrmGpu::checkCrtcAssignment(QVector<DrmConnector *> connectors, const QVect
             auto crtcsLeft = crtcs;
             crtcsLeft.removeOne(crtc);
             pipeline->setCrtc(crtc);
-            if (checkCrtcAssignment(connectors, crtcsLeft)) {
-                return true;
-            }
+            do {
+                if (checkCrtcAssignment(connectors, crtcsLeft)) {
+                    return true;
+                }
+            } while (pipeline->pruneModifier());
         }
     }
     return false;
diff --git a/src/backends/drm/drm_pipeline.cpp b/src/backends/drm/drm_pipeline.cpp
index 6cdfd29423..8d3bc34200 100644
--- a/src/backends/drm/drm_pipeline.cpp
+++ b/src/backends/drm/drm_pipeline.cpp
@@ -409,20 +409,23 @@ DrmOutput *DrmPipeline::output() const
     return m_output;
 }
 
-static const QMap<uint32_t, QVector<uint64_t>> legacyFormats = {
-    {DRM_FORMAT_XRGB8888, {}}};
-
 QMap<uint32_t, QVector<uint64_t>> DrmPipeline::formats() const
 {
-    if (m_pending.crtc) {
-        if (m_pending.crtc->primaryPlane()) {
-            return m_pending.crtc->primaryPlane()->formats();
-        } else {
-            return legacyFormats;
-        }
-    } else {
-        return {};
+    return m_pending.formats;
+}
+
+bool DrmPipeline::pruneModifier()
+{
+    if (m_pending.layer->currentBuffer()->modifier() == DRM_FORMAT_MOD_NONE
+        || m_pending.layer->currentBuffer()->modifier() == DRM_FORMAT_MOD_INVALID) {
+        return false;
     }
+    auto &modifiers = m_pending.formats[m_pending.layer->currentBuffer()->format()];
+    if (modifiers.count() <= 1) {
+        return false;
+    }
+    modifiers.removeOne(m_pending.layer->currentBuffer()->modifier());
+    return true;
 }
 
 bool DrmPipeline::needsModeset() const
@@ -608,9 +611,16 @@ Output::RgbRange DrmPipeline::rgbRange() const
     return m_pending.rgbRange;
 }
 
+static const QMap<uint32_t, QVector<uint64_t>> legacyFormats = {{DRM_FORMAT_XRGB8888, {}}};
+
 void DrmPipeline::setCrtc(DrmCrtc *crtc)
 {
     m_pending.crtc = crtc;
+    if (crtc) {
+        m_pending.formats = crtc->primaryPlane() ? crtc->primaryPlane()->formats() : legacyFormats;
+    } else {
+        m_pending.formats = {};
+    }
 }
 
 void DrmPipeline::setMode(const QSharedPointer<DrmConnectorMode> &mode)
diff --git a/src/backends/drm/drm_pipeline.h b/src/backends/drm/drm_pipeline.h
index d88c19b211..a2d27f9763 100644
--- a/src/backends/drm/drm_pipeline.h
+++ b/src/backends/drm/drm_pipeline.h
@@ -85,6 +85,7 @@ public:
     QSize bufferSize() const;
 
     QMap<uint32_t, QVector<uint64_t>> formats() const;
+    bool pruneModifier();
 
     void setOutput(DrmOutput *output);
     DrmOutput *output() const;
@@ -158,6 +159,7 @@ private:
     struct State
     {
         DrmCrtc *crtc = nullptr;
+        QMap<uint32_t, QVector<uint64_t>> formats;
         bool active = true; // whether or not the pipeline should be currently used
         bool enabled = true; // whether or not the pipeline needs a crtc
         QSharedPointer<DrmConnectorMode> mode;