backends/drm: support hardware rotation with direct scanout

This improves latency and power use for fullscreen windows when the output
is rotated

src/backends/drm/drm_egl_layer.cpp

@@ -10,6 +10,7 @@
 #include "core/iccprofile.h"
 #include "drm_backend.h"
 #include "drm_buffer.h"
+#include "drm_crtc.h"
 #include "drm_egl_backend.h"
 #include "drm_gpu.h"
 #include "drm_output.h"
@@ -64,7 +65,7 @@ std::shared_ptr<GLTexture> EglGbmLayer::texture() const
 {
     if (m_scanoutBuffer) {
         const auto ret = m_surface.eglBackend()->importDmaBufAsTexture(*m_scanoutBuffer->buffer()->dmabufAttributes());
-        ret->setContentTransform(m_pipeline->output()->transform().combine(OutputTransform::FlipY));
+        ret->setContentTransform(m_scanoutBufferTransform.combine(OutputTransform::FlipY));
         return ret;
     } else {
         return m_surface.texture();
@@ -93,7 +94,9 @@ bool EglGbmLayer::scanout(SurfaceItem *surfaceItem)
         return false;
     }
     const auto surface = item->surface();
-    if (surface->bufferTransform() != m_pipeline->output()->transform()) {
+    const auto neededTransform = surface->bufferTransform().combine(m_pipeline->output()->transform().inverted());
+    const auto plane = m_pipeline->crtc()->primaryPlane();
+    if (neededTransform != OutputTransform::Kind::Normal && (!plane || !plane->supportsTransformation(neededTransform))) {
         return false;
     }
     const auto buffer = surface->buffer();
@@ -118,6 +121,8 @@ bool EglGbmLayer::scanout(SurfaceItem *surfaceItem)
     if (!formats[dmabufAttributes->format].contains(dmabufAttributes->modifier)) {
         return false;
     }
+    m_scanoutTransform = neededTransform;
+    m_scanoutBufferTransform = surface->bufferTransform();
     m_scanoutBuffer = m_pipeline->gpu()->importBuffer(buffer, FileDescriptor{});
     if (m_scanoutBuffer && m_pipeline->testScanout()) {
         m_dmabufFeedback.scanoutSuccessful(surface);
@@ -149,4 +154,9 @@ std::chrono::nanoseconds EglGbmLayer::queryRenderTime() const
 {
     return m_surface.queryRenderTime();
 }
+
+OutputTransform EglGbmLayer::hardwareTransform() const
+{
+    return m_scanoutBuffer ? m_scanoutTransform : OutputTransform::Normal;
+}
 }

src/backends/drm/drm_egl_layer.h

@@ -38,9 +38,14 @@ public:
     ColorDescription colorDescription() const;
     void releaseBuffers() override;
     std::chrono::nanoseconds queryRenderTime() const override;
+    OutputTransform hardwareTransform() const override;
 
 private:
     std::shared_ptr<DrmFramebuffer> m_scanoutBuffer;
+    // the transform the drm plane will apply to the buffer
+    OutputTransform m_scanoutTransform = OutputTransform::Kind::Normal;
+    // the output transform the buffer is made for
+    OutputTransform m_scanoutBufferTransform = OutputTransform::Kind::Normal;
     QRegion m_currentDamage;
 
     EglGbmLayerSurface m_surface;

src/backends/drm/drm_layer.cpp

@@ -31,4 +31,9 @@ DrmPipelineLayer::DrmPipelineLayer(DrmPipeline *pipeline)
     : m_pipeline(pipeline)
 {
 }
+
+OutputTransform DrmPipelineLayer::hardwareTransform() const
+{
+    return OutputTransform::Kind::Normal;
+}
 }

src/backends/drm/drm_layer.h

@@ -38,6 +38,7 @@ public:
 
     virtual bool checkTestBuffer() = 0;
     virtual std::shared_ptr<DrmFramebuffer> currentBuffer() const = 0;
+    virtual OutputTransform hardwareTransform() const;
 
 protected:
     DrmPipeline *const m_pipeline;

src/backends/drm/drm_pipeline.cpp

@@ -256,7 +256,17 @@ DrmPipeline::Error DrmPipeline::prepareAtomicPresentation(DrmAtomicCommit *commi
         return Error::InvalidArguments;
     }
     const auto primary = m_pending.crtc->primaryPlane();
-    primary->set(commit, QPoint(0, 0), fb->buffer()->size(), centerBuffer(fb->buffer()->size(), m_pending.mode->size()));
+    const auto transform = m_primaryLayer->hardwareTransform();
+    const auto planeTransform = DrmPlane::outputTransformToPlaneTransform(transform);
+    if (primary->rotation.isValid()) {
+        if (!primary->rotation.hasEnum(planeTransform)) {
+            return Error::InvalidArguments;
+        }
+        commit->addEnum(primary->rotation, planeTransform);
+    } else if (planeTransform != DrmPlane::Transformation::Rotate0) {
+        return Error::InvalidArguments;
+    }
+    primary->set(commit, QPoint(0, 0), fb->buffer()->size(), centerBuffer(transform.map(fb->buffer()->size()), m_pending.mode->size()));
     commit->addBuffer(m_pending.crtc->primaryPlane(), fb);
     if (fb->buffer()->dmabufAttributes()->format == DRM_FORMAT_NV12) {
         if (!primary->colorEncoding.isValid() || !primary->colorRange.isValid()) {

src/backends/drm/drm_plane.cpp

@@ -172,6 +172,35 @@ void DrmPlane::releaseCurrentBuffer()
         m_current->releaseBuffer();
     }
 }
+
+DrmPlane::Transformations DrmPlane::outputTransformToPlaneTransform(OutputTransform transform)
+{
+    // note that drm transformations are counter clockwise
+    switch (transform.kind()) {
+    case OutputTransform::Kind::Normal:
+        return Transformation::Rotate0;
+    case OutputTransform::Kind::Rotate90:
+        return Transformation::Rotate270;
+    case OutputTransform::Kind::Rotate180:
+        return Transformation::Rotate180;
+    case OutputTransform::Kind::Rotate270:
+        return Transformation::Rotate90;
+    case OutputTransform::Kind::FlipY:
+        return Transformation::Rotate0 | Transformation::ReflectY;
+    case OutputTransform::Kind::FlipY90:
+        return Transformation::Rotate270 | Transformation::ReflectY;
+    case OutputTransform::Kind::FlipY180:
+        return Transformation::Rotate180 | Transformation::ReflectY;
+    case OutputTransform::Kind::FlipY270:
+        return Transformation::Rotate90 | Transformation::ReflectY;
+    }
+    Q_UNREACHABLE();
+}
+
+bool DrmPlane::supportsTransformation(OutputTransform transform) const
+{
+    return rotation.isValid() && rotation.hasEnum(outputTransformToPlaneTransform(transform));
+}
 }
 
 #include "moc_drm_plane.cpp"

src/backends/drm/drm_plane.h

@@ -9,6 +9,7 @@
 */
 #pragma once
 
+#include "core/output.h"
 #include "drm_object.h"
 
 #include <QMap>
@@ -34,6 +35,7 @@ public:
 
     bool isCrtcSupported(int pipeIndex) const;
     QMap<uint32_t, QList<uint64_t>> formats() const;
+    bool supportsTransformation(OutputTransform transform) const;
 
     std::shared_ptr<DrmFramebuffer> currentBuffer() const;
     void setCurrentBuffer(const std::shared_ptr<DrmFramebuffer> &b);
@@ -56,6 +58,7 @@ public:
     };
     Q_ENUM(Transformation)
     Q_DECLARE_FLAGS(Transformations, Transformation)
+    static Transformations outputTransformToPlaneTransform(OutputTransform transform);
     enum class PixelBlendMode : uint64_t {
         None,
         PreMultiplied,