From 883dbfda172e4b11081aaac4b590bb872914a05b Mon Sep 17 00:00:00 2001
From: Aleix Pol <aleixpol@kde.org>
Date: Fri, 10 Sep 2021 19:12:27 +0200
Subject: [PATCH] drm: Properly check for the need for transformation

At the moment we were checking for size difference between the buffers
which works for rotation but not for flipped or up-side down screens.

This changes how we check it by comparing the effective transform vs the
supported one in drm to see if they differ.

Fixes https://invent.kde.org/plasma/plasma-phone-components/-/issues/112
---
 src/plugins/platforms/drm/drm_abstract_output.h  | 1 +
 src/plugins/platforms/drm/drm_output.cpp         | 5 +++++
 src/plugins/platforms/drm/drm_output.h           | 1 +
 src/plugins/platforms/drm/drm_virtual_output.h   | 3 +++
 src/plugins/platforms/drm/egl_gbm_backend.cpp    | 7 +++----
 src/plugins/platforms/drm/egl_stream_backend.cpp | 4 ++--
 6 files changed, 15 insertions(+), 6 deletions(-)

diff --git a/src/plugins/platforms/drm/drm_abstract_output.h b/src/plugins/platforms/drm/drm_abstract_output.h
index e1c9a54595..a48a744147 100644
--- a/src/plugins/platforms/drm/drm_abstract_output.h
+++ b/src/plugins/platforms/drm/drm_abstract_output.h
@@ -31,6 +31,7 @@ public:
 
     virtual bool present(const QSharedPointer<DrmBuffer> &buffer, QRegion damagedRegion) = 0;
 
+    virtual bool needsSoftwareTransformation() const = 0;
     virtual bool isDpmsEnabled() const = 0;
     virtual GbmBuffer *currentBuffer() const = 0;
     virtual QSize sourceSize() const = 0;
diff --git a/src/plugins/platforms/drm/drm_output.cpp b/src/plugins/platforms/drm/drm_output.cpp
index e654a4d754..907e33975b 100644
--- a/src/plugins/platforms/drm/drm_output.cpp
+++ b/src/plugins/platforms/drm/drm_output.cpp
@@ -346,6 +346,11 @@ void DrmOutput::updateModes()
     }
 }
 
+bool DrmOutput::needsSoftwareTransformation() const
+{
+    return m_pipeline->transformation() != outputToPlaneTransform(transform());
+}
+
 void DrmOutput::updateMode(const QSize &size, uint32_t refreshRate)
 {
     auto conn = m_pipeline->connector();
diff --git a/src/plugins/platforms/drm/drm_output.h b/src/plugins/platforms/drm/drm_output.h
index 2c8d26c638..868e83002e 100644
--- a/src/plugins/platforms/drm/drm_output.h
+++ b/src/plugins/platforms/drm/drm_output.h
@@ -58,6 +58,7 @@ public:
     QSize sourceSize() const override;
     bool isFormatSupported(uint32_t drmFormat) const override;
     QVector<uint64_t> supportedModifiers(uint32_t drmFormat) const override;
+    bool needsSoftwareTransformation() const override;
 
 private:
     friend class DrmGpu;
diff --git a/src/plugins/platforms/drm/drm_virtual_output.h b/src/plugins/platforms/drm/drm_virtual_output.h
index 9b2b2887b4..ac967f2398 100644
--- a/src/plugins/platforms/drm/drm_virtual_output.h
+++ b/src/plugins/platforms/drm/drm_virtual_output.h
@@ -42,6 +42,9 @@ public:
         Q_UNUSED(gamma);
         return true;
     }
+    bool needsSoftwareTransformation() const override {
+        return false;
+    }
 
 private:
     void vblank(std::chrono::nanoseconds timestamp);
diff --git a/src/plugins/platforms/drm/egl_gbm_backend.cpp b/src/plugins/platforms/drm/egl_gbm_backend.cpp
index d13723af8f..0edcb99ed0 100644
--- a/src/plugins/platforms/drm/egl_gbm_backend.cpp
+++ b/src/plugins/platforms/drm/egl_gbm_backend.cpp
@@ -179,7 +179,7 @@ bool EglGbmBackend::resetOutput(Output &output)
     output.current = {};
     output.current.gbmSurface = gbmSurface;
 
-    if (size == output.output->pixelSize()) {
+    if (!output.output->needsSoftwareTransformation())  {
         output.current.shadowBuffer = nullptr;
     } else {
         makeContextCurrent(output.current);
@@ -524,10 +524,9 @@ bool EglGbmBackend::doesRenderFit(DrmAbstractOutput *output, const Output::Rende
     if (surfaceSize != render.gbmSurface->size()) {
         return false;
     }
-    QSize pixelSize = output->pixelSize();
-    bool needsTexture = surfaceSize != pixelSize;
+    bool needsTexture = output->needsSoftwareTransformation();
     if (needsTexture) {
-        return render.shadowBuffer && render.shadowBuffer->textureSize() == pixelSize;
+        return render.shadowBuffer && render.shadowBuffer->textureSize() == output->pixelSize();
     } else {
         return render.shadowBuffer == nullptr;
     }
diff --git a/src/plugins/platforms/drm/egl_stream_backend.cpp b/src/plugins/platforms/drm/egl_stream_backend.cpp
index 578b902a2e..6761def821 100644
--- a/src/plugins/platforms/drm/egl_stream_backend.cpp
+++ b/src/plugins/platforms/drm/egl_stream_backend.cpp
@@ -358,7 +358,7 @@ bool EglStreamBackend::resetOutput(Output &o)
         o.eglStream = stream;
         o.eglSurface = eglSurface;
 
-        if (sourceSize != drmOutput->pixelSize()) {
+        if (drmOutput->needsSoftwareTransformation()) {
             makeContextCurrent(o);
             o.shadowBuffer = QSharedPointer<ShadowBuffer>::create(o.output->pixelSize());
             if (!o.shadowBuffer->isComplete()) {
@@ -482,7 +482,7 @@ bool EglStreamBackend::needsReset(const Output &o) const
     if (surfaceSize != o.output->sourceSize()) {
         return true;
     }
-    bool needsTexture = surfaceSize != o.output->pixelSize();
+    bool needsTexture = o.output->needsSoftwareTransformation();
     if (needsTexture) {
         return !o.shadowBuffer || o.shadowBuffer->textureSize() != o.output->pixelSize();
     } else {