From 420358709a0a725b8701d461c2a96e3f106c8e4b Mon Sep 17 00:00:00 2001
From: Xaver Hugl <xaver.hugl@gmail.com>
Date: Sat, 14 May 2022 23:35:27 +0200
Subject: [PATCH] backends/drm: don't test hardware rotation if not supported
 by the driver

---
 src/backends/drm/drm_pipeline.cpp | 13 +++++++++++++
 src/backends/drm/drm_pipeline.h   |  1 +
 2 files changed, 14 insertions(+)

diff --git a/src/backends/drm/drm_pipeline.cpp b/src/backends/drm/drm_pipeline.cpp
index 79388f279f..76534864ab 100644
--- a/src/backends/drm/drm_pipeline.cpp
+++ b/src/backends/drm/drm_pipeline.cpp
@@ -110,6 +110,7 @@ DrmPipeline::Error DrmPipeline::commitPipelinesAtomic(const QVector<DrmPipeline
         return Error::OutofMemory;
     }
     for (const auto &pipeline : pipelines) {
+        pipeline->checkHardwareRotation();
         if (pipeline->activePending()) {
             if (!pipeline->m_pending.layer->checkTestBuffer()) {
                 qCWarning(KWIN_DRM) << "Checking test buffer failed for" << mode;
@@ -290,6 +291,18 @@ bool DrmPipeline::populateAtomicValues(drmModeAtomicReq *req)
     return true;
 }
 
+void DrmPipeline::checkHardwareRotation()
+{
+    if (m_pending.crtc && m_pending.crtc->primaryPlane()) {
+        const bool supported = (m_pending.bufferOrientation & m_pending.crtc->primaryPlane()->supportedTransformations());
+        if (!supported) {
+            m_pending.bufferOrientation = DrmPlane::Transformations(DrmPlane::Transformation::Rotate0);
+        }
+    } else {
+        m_pending.bufferOrientation = DrmPlane::Transformation::Rotate0;
+    }
+}
+
 uint32_t DrmPipeline::calculateUnderscan()
 {
     const auto size = m_pending.mode->size();
diff --git a/src/backends/drm/drm_pipeline.h b/src/backends/drm/drm_pipeline.h
index 17e1f0919a..7e63ee0a53 100644
--- a/src/backends/drm/drm_pipeline.h
+++ b/src/backends/drm/drm_pipeline.h
@@ -141,6 +141,7 @@ private:
     bool isBufferForDirectScanout() const;
     uint32_t calculateUnderscan();
     static Error errnoToError();
+    void checkHardwareRotation();
 
     // legacy only
     Error presentLegacy();