backends/drm: fail atomic tests gracefully when buffer allocations fail

This should fix the crash. However, it's still unclear to me why allocations
fail in the first place

CCBUG: 452572

src/backends/drm/drm_layer.h

@@ -36,12 +36,7 @@ class DrmPipelineLayer : public DrmOutputLayer
 public:
     DrmPipelineLayer(DrmPipeline *pipeline);
 
-    /**
-     * @returns a buffer for atomic test commits
-     * If no fitting buffer is available, a new current buffer is created
-     */
-    virtual QSharedPointer<DrmBuffer> testBuffer() = 0;
-
+    virtual bool checkTestBuffer() = 0;
     virtual QSharedPointer<DrmBuffer> currentBuffer() const = 0;
     virtual bool hasDirectScanoutBuffer() const;
 

src/backends/drm/drm_lease_egl_gbm_layer.cpp

@@ -27,7 +27,7 @@ DrmLeaseEglGbmLayer::DrmLeaseEglGbmLayer(EglGbmBackend *backend, DrmPipeline *pi
     });
 }
 
-QSharedPointer<DrmBuffer> DrmLeaseEglGbmLayer::testBuffer()
+bool DrmLeaseEglGbmLayer::checkTestBuffer()
 {
     const auto mods = m_pipeline->formats().value(DRM_FORMAT_XRGB8888);
     const auto size = m_pipeline->bufferSize();
@@ -49,7 +49,7 @@ QSharedPointer<DrmBuffer> DrmLeaseEglGbmLayer::testBuffer()
             qCWarning(KWIN_DRM) << "Failed to create gbm_bo for lease output";
         }
     }
-    return m_buffer;
+    return m_buffer && m_buffer->bufferId() != 0;
 }
 
 QSharedPointer<DrmBuffer> DrmLeaseEglGbmLayer::currentBuffer() const

src/backends/drm/drm_lease_egl_gbm_layer.h

@@ -24,7 +24,7 @@ public:
     OutputLayerBeginFrameInfo beginFrame() override;
     void endFrame(const QRegion &damagedRegion, const QRegion &renderedRegion) override;
 
-    QSharedPointer<DrmBuffer> testBuffer() override;
+    bool checkTestBuffer() override;
     QSharedPointer<DrmBuffer> currentBuffer() const override;
 
 private:

src/backends/drm/drm_pipeline.cpp

@@ -113,7 +113,7 @@ bool DrmPipeline::commitPipelinesAtomic(const QVector<DrmPipeline *> &pipelines,
         return false;
     };
     for (const auto &pipeline : pipelines) {
-        if (!pipeline->pending.layer->testBuffer()) {
+        if (!pipeline->pending.layer->checkTestBuffer()) {
             qCWarning(KWIN_DRM) << "Checking test buffer failed for" << mode;
             return failed();
         }

src/backends/drm/drm_pipeline_legacy.cpp

@@ -38,7 +38,7 @@ bool DrmPipeline::presentLegacy()
 bool DrmPipeline::legacyModeset()
 {
     uint32_t connId = m_connector->id();
-    if (!pending.layer->testBuffer() || drmModeSetCrtc(gpu()->fd(), pending.crtc->id(), pending.layer->currentBuffer()->bufferId(), 0, 0, &connId, 1, pending.mode->nativeMode()) != 0) {
+    if (!pending.layer->checkTestBuffer() || drmModeSetCrtc(gpu()->fd(), pending.crtc->id(), pending.layer->currentBuffer()->bufferId(), 0, 0, &connId, 1, pending.mode->nativeMode()) != 0) {
         qCWarning(KWIN_DRM) << "Modeset failed!" << strerror(errno);
         pending = m_next;
         return false;

src/backends/drm/drm_qpainter_layer.cpp

@@ -52,12 +52,12 @@ void DrmQPainterLayer::endFrame(const QRegion &renderedRegion, const QRegion &da
     m_swapchain->releaseBuffer(m_swapchain->currentBuffer(), damagedRegion);
 }
 
-QSharedPointer<DrmBuffer> DrmQPainterLayer::testBuffer()
+bool DrmQPainterLayer::checkTestBuffer()
 {
     if (!doesSwapchainFit()) {
         m_swapchain = QSharedPointer<DumbSwapchain>::create(m_pipeline->gpu(), m_pipeline->bufferSize(), DRM_FORMAT_XRGB8888);
     }
-    return m_swapchain->currentBuffer();
+    return !m_swapchain->isEmpty();
 }
 
 bool DrmQPainterLayer::doesSwapchainFit() const
@@ -110,13 +110,13 @@ DrmLeaseQPainterLayer::DrmLeaseQPainterLayer(DrmQPainterBackend *backend, DrmPip
     });
 }
 
-QSharedPointer<DrmBuffer> DrmLeaseQPainterLayer::testBuffer()
+bool DrmLeaseQPainterLayer::checkTestBuffer()
 {
     const auto size = m_pipeline->bufferSize();
     if (!m_buffer || m_buffer->size() != size) {
         m_buffer = QSharedPointer<DrmDumbBuffer>::create(m_pipeline->gpu(), size, DRM_FORMAT_XRGB8888);
     }
-    return m_buffer;
+    return m_buffer->bufferId() != 0;
 }
 
 QSharedPointer<DrmBuffer> DrmLeaseQPainterLayer::currentBuffer() const

src/backends/drm/drm_qpainter_layer.h

@@ -27,7 +27,7 @@ public:
 
     OutputLayerBeginFrameInfo beginFrame() override;
     void endFrame(const QRegion &damagedRegion, const QRegion &renderedRegion) override;
-    QSharedPointer<DrmBuffer> testBuffer() override;
+    bool checkTestBuffer() override;
     QSharedPointer<DrmBuffer> currentBuffer() const override;
     QRegion currentDamage() const override;
 
@@ -62,7 +62,7 @@ public:
     OutputLayerBeginFrameInfo beginFrame() override;
     void endFrame(const QRegion &damagedRegion, const QRegion &renderedRegion) override;
 
-    QSharedPointer<DrmBuffer> testBuffer() override;
+    bool checkTestBuffer() override;
     QSharedPointer<DrmBuffer> currentBuffer() const override;
 
 private:

src/backends/drm/egl_gbm_layer.cpp

@@ -34,17 +34,6 @@ EglGbmLayer::EglGbmLayer(EglGbmBackend *eglBackend, DrmPipeline *pipeline)
     , m_surface(pipeline->gpu(), eglBackend)
     , m_dmabufFeedback(pipeline->gpu(), eglBackend)
 {
-    connect(eglBackend, &EglGbmBackend::aboutToBeDestroyed, this, &EglGbmLayer::destroyResources);
-}
-
-EglGbmLayer::~EglGbmLayer()
-{
-    destroyResources();
-}
-
-void EglGbmLayer::destroyResources()
-{
-    m_surface.destroyResources();
 }
 
 OutputLayerBeginFrameInfo EglGbmLayer::beginFrame()
@@ -74,21 +63,17 @@ QRegion EglGbmLayer::currentDamage() const
     return m_currentDamage;
 }
 
-QSharedPointer<DrmBuffer> EglGbmLayer::testBuffer()
+bool EglGbmLayer::checkTestBuffer()
 {
     if (!m_surface.doesSurfaceFit(m_pipeline->bufferSize(), m_pipeline->formats())) {
-        renderTestBuffer();
+        const auto buffer = m_surface.renderTestBuffer(m_pipeline->bufferSize(), m_pipeline->formats());
+        if (!buffer) {
+            return false;
+        } else {
+            m_currentBuffer = buffer;
+        }
     }
-    return m_currentBuffer;
-}
-
-bool EglGbmLayer::renderTestBuffer()
-{
-    const auto oldBuffer = m_currentBuffer;
-    beginFrame();
-    glClear(GL_COLOR_BUFFER_BIT);
-    endFrame(QRegion(), infiniteRegion());
-    return m_currentBuffer != oldBuffer;
+    return true;
 }
 
 QSharedPointer<GLTexture> EglGbmLayer::texture() const

src/backends/drm/egl_gbm_layer.h

@@ -35,22 +35,18 @@ class EglGbmLayer : public DrmPipelineLayer
 {
 public:
     EglGbmLayer(EglGbmBackend *eglBackend, DrmPipeline *pipeline);
-    ~EglGbmLayer();
 
     OutputLayerBeginFrameInfo beginFrame() override;
     void aboutToStartPainting(const QRegion &damagedRegion) override;
     void endFrame(const QRegion &renderedRegion, const QRegion &damagedRegion) override;
     bool scanout(SurfaceItem *surfaceItem) override;
-    QSharedPointer<DrmBuffer> testBuffer() override;
+    bool checkTestBuffer() override;
     QSharedPointer<DrmBuffer> currentBuffer() const override;
     bool hasDirectScanoutBuffer() const override;
     QRegion currentDamage() const override;
     QSharedPointer<GLTexture> texture() const override;
 
 private:
-    bool renderTestBuffer();
-    void destroyResources();
-
     QSharedPointer<DrmGbmBuffer> m_scanoutBuffer;
     QSharedPointer<DrmBuffer> m_currentBuffer;
     QRegion m_currentDamage;

src/backends/drm/egl_gbm_layer_surface.cpp

@@ -54,21 +54,8 @@ void EglGbmLayerSurface::destroyResources()
 
 OutputLayerBeginFrameInfo EglGbmLayerSurface::startRendering(const QSize &bufferSize, DrmPlane::Transformations renderOrientation, DrmPlane::Transformations bufferOrientation, const QMap<uint32_t, QVector<uint64_t>> &formats)
 {
-    // gbm surface
-    if (doesGbmSurfaceFit(m_gbmSurface.data(), bufferSize, formats)) {
-        m_oldGbmSurface.reset();
-    } else {
-        if (doesGbmSurfaceFit(m_oldGbmSurface.data(), bufferSize, formats)) {
-            m_gbmSurface = m_oldGbmSurface;
-        } else {
-            if (!createGbmSurface(bufferSize, formats)) {
-                return {};
-            }
-            // dmabuf might work with the new surface
-            m_importMode = MultiGpuImportMode::Dmabuf;
-            m_importSwapchain.reset();
-            m_oldImportSwapchain.reset();
-        }
+    if (!checkGbmSurface(bufferSize, formats)) {
+        return {};
     }
     if (!m_gbmSurface->makeContextCurrent()) {
         return {};
@@ -147,6 +134,26 @@ std::optional<std::tuple<QSharedPointer<DrmBuffer>, QRegion>> EglGbmLayerSurface
     }
 }
 
+bool EglGbmLayerSurface::checkGbmSurface(const QSize &bufferSize, const QMap<uint32_t, QVector<uint64_t>> &formats)
+{
+    if (doesGbmSurfaceFit(m_gbmSurface.data(), bufferSize, formats)) {
+        m_oldGbmSurface.reset();
+    } else {
+        if (doesGbmSurfaceFit(m_oldGbmSurface.data(), bufferSize, formats)) {
+            m_gbmSurface = m_oldGbmSurface;
+        } else {
+            if (!createGbmSurface(bufferSize, formats)) {
+                return false;
+            }
+            // dmabuf might work with the new surface
+            m_importMode = MultiGpuImportMode::Dmabuf;
+            m_importSwapchain.reset();
+            m_oldImportSwapchain.reset();
+        }
+    }
+    return m_gbmSurface != nullptr;
+}
+
 bool EglGbmLayerSurface::createGbmSurface(const QSize &size, uint32_t format, const QVector<uint64_t> &modifiers)
 {
     static bool modifiersEnvSet = false;
@@ -374,4 +381,20 @@ QSharedPointer<GLTexture> EglGbmLayerSurface::texture() const
     }
     return gbmBuffer->createTexture(m_eglBackend->eglDisplay());
 }
+
+QSharedPointer<DrmBuffer> EglGbmLayerSurface::renderTestBuffer(const QSize &bufferSize, const QMap<uint32_t, QVector<uint64_t>> &formats)
+{
+    if (!checkGbmSurface(bufferSize, formats) || !m_gbmSurface->makeContextCurrent()) {
+        return nullptr;
+    }
+    glClear(GL_COLOR_BUFFER_BIT);
+    if (m_gpu == m_eglBackend->gpu()) {
+        return m_gbmSurface->swapBuffersForDrm(infiniteRegion());
+    } else {
+        if (m_gbmSurface->swapBuffers(infiniteRegion())) {
+            return importBuffer();
+        }
+    }
+    return nullptr;
+}
 }

src/backends/drm/egl_gbm_layer_surface.h

@@ -49,8 +49,10 @@ public:
     QSharedPointer<GLTexture> texture() const;
     void destroyResources();
     EglGbmBackend *eglBackend() const;
+    QSharedPointer<DrmBuffer> renderTestBuffer(const QSize &bufferSize, const QMap<uint32_t, QVector<uint64_t>> &formats);
 
 private:
+    bool checkGbmSurface(const QSize &size, const QMap<uint32_t, QVector<uint64_t>> &formats);
     bool createGbmSurface(const QSize &size, uint32_t format, const QVector<uint64_t> &modifiers);
     bool createGbmSurface(const QSize &size, const QMap<uint32_t, QVector<uint64_t>> &formats);
     bool doesGbmSurfaceFit(GbmSurface *surf, const QSize &size, const QMap<uint32_t, QVector<uint64_t>> &formats) const;