diff --git a/src/backends/drm/drm_pipeline.cpp b/src/backends/drm/drm_pipeline.cpp
index 17e85e3def..0b6400bff5 100644
--- a/src/backends/drm/drm_pipeline.cpp
+++ b/src/backends/drm/drm_pipeline.cpp
@@ -30,8 +30,9 @@
 namespace KWin
 {
 
-static const QMap<uint32_t, QVector<uint64_t>> legacyFormats = {{DRM_FORMAT_XRGB8888, {}}};
-static const QMap<uint32_t, QVector<uint64_t>> legacyCursorFormats = {{DRM_FORMAT_ARGB8888, {}}};
+static const QVector<uint64_t> implicitModifier = {DRM_FORMAT_MOD_INVALID};
+static const QMap<uint32_t, QVector<uint64_t>> legacyFormats = {{DRM_FORMAT_XRGB8888, implicitModifier}};
+static const QMap<uint32_t, QVector<uint64_t>> legacyCursorFormats = {{DRM_FORMAT_ARGB8888, implicitModifier}};
 
 DrmPipeline::DrmPipeline(DrmConnector *conn)
     : m_connector(conn)
@@ -450,16 +451,14 @@ QMap<uint32_t, QVector<uint64_t>> DrmPipeline::cursorFormats() const
 bool DrmPipeline::pruneModifier()
 {
     const DmaBufAttributes *dmabufAttributes = m_pending.layer->currentBuffer() ? m_pending.layer->currentBuffer()->buffer()->dmabufAttributes() : nullptr;
-    if (!dmabufAttributes
-        || dmabufAttributes->modifier == DRM_FORMAT_MOD_NONE
-        || dmabufAttributes->modifier == DRM_FORMAT_MOD_INVALID) {
+    if (!dmabufAttributes) {
         return false;
     }
     auto &modifiers = m_pending.formats[dmabufAttributes->format];
-    if (modifiers.empty()) {
+    if (modifiers == implicitModifier) {
         return false;
     } else {
-        modifiers.clear();
+        modifiers = implicitModifier;
         return true;
     }
 }
diff --git a/src/backends/drm/drm_plane.cpp b/src/backends/drm/drm_plane.cpp
index b77ab1d8d6..8e9fcd56c6 100644
--- a/src/backends/drm/drm_plane.cpp
+++ b/src/backends/drm/drm_plane.cpp
@@ -95,13 +95,14 @@ bool DrmPlane::updateProperties()
             m_supportedFormats[iterator.fmt].push_back(iterator.mod);
         }
     } else {
+        const QVector<uint64_t> modifiers = {DRM_FORMAT_MOD_INVALID};
         for (uint32_t i = 0; i < p->count_formats; i++) {
-            m_supportedFormats.insert(p->formats[i], {DRM_FORMAT_MOD_LINEAR});
+            m_supportedFormats.insert(p->formats[i], modifiers);
+        }
+        if (m_supportedFormats.isEmpty()) {
+            qCWarning(KWIN_DRM) << "Driver doesn't advertise any formats for this plane. Falling back to XRGB8888 without explicit modifiers";
+            m_supportedFormats.insert(DRM_FORMAT_XRGB8888, modifiers);
         }
-    }
-    if (m_supportedFormats.isEmpty()) {
-        qCWarning(KWIN_DRM) << "Driver doesn't advertise any formats for this plane. Falling back to XRGB8888 without explicit modifiers";
-        m_supportedFormats.insert(DRM_FORMAT_XRGB8888, {});
     }
     return true;
 }