backends/drm: test crtc+connector combinations with hardware rotation

This should enable KWin to use hardware rotation in more situations. As
a fallback all hardware rotation is disabled and the test is done again

src/backends/drm/drm_gpu.cpp

@@ -393,10 +393,23 @@ bool DrmGpu::testPendingConfiguration(TestMode mode)
             return c1->getProp(DrmConnector::PropertyIndex::CrtcId)->current() > c2->getProp(DrmConnector::PropertyIndex::CrtcId)->current();
         });
     }
-    if (mode == TestMode::TestWithCrtcReallocation) {
+    const auto &test = [&connectors, &crtcs, this, mode](){
-        return checkCrtcAssignment(connectors, crtcs);
+        if (mode == TestMode::TestWithCrtcReallocation) {
+            return checkCrtcAssignment(connectors, crtcs);
+        } else {
+            return testPipelines();
+        }
+    };
+    if (test()) {
+        return true;
     } else {
-        return testPipelines();
+        // try again without hw rotation
+        bool hwRotationUsed = false;
+        for (const auto &pipeline : qAsConst(m_pipelines)) {
+            hwRotationUsed |= (pipeline->pending.transformation != DrmPlane::Transformations(DrmPlane::Transformation::Rotate0));
+            pipeline->pending.transformation = DrmPlane::Transformation::Rotate0;
+        }
+        return hwRotationUsed ? test() : false;
     }
 }
 

src/backends/drm/drm_output.cpp

@@ -290,22 +290,6 @@ DrmPlane::Transformations outputToPlaneTransform(DrmOutput::Transform transform)
     }
 }
 
-void DrmOutput::updateTransform(Transform transform)
-{
-    setTransformInternal(transform);
-    static bool valid;
-    // If not set or wrong value, assume KWIN_DRM_SW_ROTATIONS_ONLY=1 until DRM transformations are reliable
-    static int envOnlySoftwareRotations = qEnvironmentVariableIntValue("KWIN_DRM_SW_ROTATIONS_ONLY", &valid) != 0;
-    if (valid && !envOnlySoftwareRotations) {
-        m_pipeline->pending.transformation = outputToPlaneTransform(transform);
-        if (m_gpu->testPendingConfiguration(DrmGpu::TestMode::TestOnly)) {
-            m_pipeline->applyPendingChanges();
-        } else {
-            m_pipeline->revertPendingChanges();
-        }
-    }
-}
-
 void DrmOutput::updateModes()
 {
     auto conn = m_pipeline->connector();
@@ -411,6 +395,9 @@ QVector<uint64_t> DrmOutput::supportedModifiers(uint32_t drmFormat) const
 
 bool DrmOutput::queueChanges(const WaylandOutputConfig &config)
 {
+    static bool valid;
+    static int envOnlySoftwareRotations = qEnvironmentVariableIntValue("KWIN_DRM_SW_ROTATIONS_ONLY", &valid) == 1 || !valid;
+
     auto props = config.constChangeSet(this);
     m_pipeline->pending.active = props->enabled;
     auto modelist = m_connector->modes();
@@ -427,7 +414,9 @@ bool DrmOutput::queueChanges(const WaylandOutputConfig &config)
     m_pipeline->pending.modeIndex = index;
     m_pipeline->pending.overscan = props->overscan;
     m_pipeline->pending.rgbRange = props->rgbRange;
-    m_pipeline->pending.transformation = DrmPlane::Transformation::Rotate0;
+    if (!envOnlySoftwareRotations) {
+        m_pipeline->pending.transformation = outputToPlaneTransform(props->transform);
+    }
     m_pipeline->pending.enabled = props->enabled;
     return true;
 }
@@ -444,7 +433,7 @@ void DrmOutput::applyQueuedChanges(const WaylandOutputConfig &config)
     setEnabled(props->enabled && m_pipeline->pending.crtc);
     moveTo(props->pos);
     setScale(props->scale);
-    updateTransform(props->transform);
+    setTransformInternal(props->transform);
 
     m_connector->setModeIndex(m_pipeline->pending.modeIndex);
     auto mode = m_connector->currentMode();

src/backends/drm/drm_output.h

@@ -72,8 +72,6 @@ private:
 
     QVector<AbstractWaylandOutput::Mode> getModes() const;
 
-    void updateTransform(Transform transform) override;
-
     int gammaRampSize() const override;
     bool setGammaRamp(const GammaRamp &gamma) override;
     void updateCursor();