From 3f3d41f25973baff371ad8a8a60ad2309410a496 Mon Sep 17 00:00:00 2001
From: Xaver Hugl <xaver.hugl@gmail.com>
Date: Sun, 30 Oct 2022 23:08:37 +0100
Subject: [PATCH] backends/drm: fix common mode generation

There were overflows in the bandwidth estimation calculation. To simplify things,
also only generate common modes if the display doesn't advertise them on its own
already.
---
 src/backends/drm/drm_connector.cpp | 23 ++++++++---------------
 src/backends/drm/drm_connector.h   |  2 +-
 2 files changed, 9 insertions(+), 16 deletions(-)

diff --git a/src/backends/drm/drm_connector.cpp b/src/backends/drm/drm_connector.cpp
index 62eef7cb19..0bbc6a382f 100644
--- a/src/backends/drm/drm_connector.cpp
+++ b/src/backends/drm/drm_connector.cpp
@@ -319,7 +319,9 @@ bool DrmConnector::updateProperties()
         }
         m_modes.clear();
         m_modes.append(m_driverModes);
-        m_modes.append(generateCommonModes());
+        if (m_driverModes.size() == 1) {
+            m_modes.append(generateCommonModes(m_driverModes.front().get()));
+        }
         if (m_pipeline->mode()) {
             if (const auto mode = findMode(*m_pipeline->mode()->nativeMode())) {
                 m_pipeline->setMode(mode);
@@ -393,23 +395,14 @@ static const QVector<QSize> s_commonModes = {
     QSize(1280, 720),
 };
 
-QList<std::shared_ptr<DrmConnectorMode>> DrmConnector::generateCommonModes()
+QList<std::shared_ptr<DrmConnectorMode>> DrmConnector::generateCommonModes(DrmConnectorMode *baseMode)
 {
     QList<std::shared_ptr<DrmConnectorMode>> ret;
-    uint32_t maxBandwidthEstimation = 0;
-    QSize maxSize;
-    for (const auto &mode : qAsConst(m_driverModes)) {
-        if (mode->size().width() > maxSize.width() || mode->size().height() > maxSize.height()) {
-            maxSize = mode->size();
-            maxBandwidthEstimation = std::max(maxBandwidthEstimation, static_cast<uint32_t>(mode->size().width() * mode->size().height() * mode->refreshRate()));
-        }
-    }
+    const QSize maxSize = baseMode->size();
+    const uint64_t maxBandwidthEstimation = maxSize.width() * maxSize.height() * uint64_t(baseMode->refreshRate());
     for (const auto &size : s_commonModes) {
-        uint32_t bandwidthEstimation = size.width() * size.height() * 60000;
-        const auto it = std::find_if(m_driverModes.constBegin(), m_driverModes.constEnd(), [size](const auto &mode) {
-            return mode->size() == size;
-        });
-        if (it == m_driverModes.constEnd() && size.width() <= maxSize.width() && size.height() <= maxSize.height() && bandwidthEstimation < maxBandwidthEstimation) {
+        const uint64_t bandwidthEstimation = size.width() * size.height() * 60000ull;
+        if (size.width() <= maxSize.width() && size.height() <= maxSize.height() && bandwidthEstimation <= maxBandwidthEstimation) {
             ret << generateMode(size, 60);
         }
     }
diff --git a/src/backends/drm/drm_connector.h b/src/backends/drm/drm_connector.h
index 54e8469104..40edb4b3fe 100644
--- a/src/backends/drm/drm_connector.h
+++ b/src/backends/drm/drm_connector.h
@@ -114,7 +114,7 @@ public:
     static DrmContentType kwinToDrmContentType(ContentType type);
 
 private:
-    QList<std::shared_ptr<DrmConnectorMode>> generateCommonModes();
+    QList<std::shared_ptr<DrmConnectorMode>> generateCommonModes(DrmConnectorMode *baseMode);
     std::shared_ptr<DrmConnectorMode> generateMode(const QSize &size, float refreshRate);
 
     std::unique_ptr<DrmPipeline> m_pipeline;