scene/workspacescene: extend direct scanout candidate search to include multiple subsurfaces

This way, for example subsurfaces for black bars behind videos can be scanned out

src/compositor_wayland.cpp

@@ -316,14 +316,14 @@ void WaylandCompositor::composite(RenderLoop *renderLoop)
         }
 
         bool directScanout = false;
-        if (const auto scanoutCandidate = superLayer->delegate()->scanoutCandidate()) {
+        if (const auto scanoutCandidates = superLayer->delegate()->scanoutCandidates(1); !scanoutCandidates.isEmpty()) {
             const auto sublayers = superLayer->sublayers();
             const bool scanoutPossible = std::none_of(sublayers.begin(), sublayers.end(), [](RenderLayer *sublayer) {
                 return sublayer->isVisible();
             });
             if (scanoutPossible) {
-                primaryLayer->setTargetRect(centerBuffer(output->transform().map(scanoutCandidate->size()), output->modeSize()));
-                directScanout = primaryLayer->attemptScanout(scanoutCandidate, frame);
+                primaryLayer->setTargetRect(centerBuffer(output->transform().map(scanoutCandidates[0]->size()), output->modeSize()));
+                directScanout = primaryLayer->attemptScanout(scanoutCandidates[0], frame);
             }
         } else {
             primaryLayer->notifyNoScanoutCandidate();

src/core/renderlayerdelegate.cpp

@@ -32,9 +32,9 @@ void RenderLayerDelegate::postPaint()
 {
 }
 
-SurfaceItem *RenderLayerDelegate::scanoutCandidate() const
+QList<SurfaceItem *> RenderLayerDelegate::scanoutCandidates(ssize_t maxCount) const
 {
-    return nullptr;
+    return {};
 }
 
 } // namespace KWin

src/core/renderlayerdelegate.h

@@ -53,7 +53,7 @@ public:
      * Returns the direct scanout candidate hint. It can be used to avoid compositing the
      * render layer.
      */
-    virtual SurfaceItem *scanoutCandidate() const;
+    virtual QList<SurfaceItem *> scanoutCandidates(ssize_t maxCount) const;
 
     /**
      * This function is called when the compositor wants the render layer delegate

src/scene/scene.cpp

@@ -24,9 +24,9 @@ SceneDelegate::~SceneDelegate()
     m_scene->removeDelegate(this);
 }
 
-SurfaceItem *SceneDelegate::scanoutCandidate() const
+QList<SurfaceItem *> SceneDelegate::scanoutCandidates(ssize_t maxCount) const
 {
-    return m_scene->scanoutCandidate();
+    return m_scene->scanoutCandidates(maxCount);
 }
 
 QRegion SceneDelegate::prePaint()
@@ -134,9 +134,9 @@ void Scene::removeDelegate(SceneDelegate *delegate)
     Q_EMIT delegateRemoved(delegate);
 }
 
-SurfaceItem *Scene::scanoutCandidate() const
+QList<SurfaceItem *> Scene::scanoutCandidates(ssize_t maxCount) const
 {
-    return nullptr;
+    return {};
 }
 
 void Scene::frame(SceneDelegate *delegate, OutputFrame *frame)

src/scene/scene.h

@@ -29,7 +29,7 @@ public:
     qreal scale() const;
     QRect viewport() const;
 
-    SurfaceItem *scanoutCandidate() const override;
+    QList<SurfaceItem *> scanoutCandidates(ssize_t maxCount) const override;
     void frame(OutputFrame *frame) override;
     QRegion prePaint() override;
     void postPaint() override;
@@ -81,7 +81,7 @@ public:
     void addDelegate(SceneDelegate *delegate);
     void removeDelegate(SceneDelegate *delegate);
 
-    virtual SurfaceItem *scanoutCandidate() const;
+    virtual QList<SurfaceItem *> scanoutCandidates(ssize_t maxCount) const;
     virtual QRegion prePaint(SceneDelegate *delegate) = 0;
     virtual void postPaint() = 0;
     virtual void paint(const RenderTarget &renderTarget, const QRegion &region) = 0;

src/scene/workspacescene.cpp

@@ -167,41 +167,55 @@ static SurfaceItem *findTopMostSurface(SurfaceItem *item)
     return item;
 }
 
-SurfaceItem *WorkspaceScene::scanoutCandidate() const
+static bool addCandidates(SurfaceItem *item, QList<SurfaceItem *> &candidates, ssize_t maxCount)
+{
+    const QList<Item *> children = item->sortedChildItems();
+    for (const auto &child : children | std::views::reverse) {
+        if (child->z() >= 0 && child->isVisible()) {
+            if (!addCandidates(static_cast<SurfaceItem *>(child), candidates, maxCount)) {
+                return false;
+            }
+        }
+    }
+    if (candidates.size() >= maxCount) {
+        return false;
+    }
+    candidates.push_back(item);
+    for (const auto &child : children | std::views::reverse) {
+        if (child->z() < 0 && child->isVisible()) {
+            if (!addCandidates(static_cast<SurfaceItem *>(child), candidates, maxCount)) {
+                return false;
+            }
+        }
+    }
+    return true;
+}
+
+QList<SurfaceItem *> WorkspaceScene::scanoutCandidates(ssize_t maxCount) const
 {
     if (!waylandServer()) {
-        return nullptr;
+        return {};
     }
-    SurfaceItem *candidate = nullptr;
+    QList<SurfaceItem *> ret;
     if (!effects->blocksDirectScanout()) {
         for (int i = stacking_order.count() - 1; i >= 0; i--) {
             WindowItem *windowItem = stacking_order[i];
             Window *window = windowItem->window();
             if (window->isOnOutput(painted_screen) && window->opacity() > 0 && windowItem->isVisible()) {
-                if (!window->isClient() || !window->isFullScreen() || window->opacity() != 1.0) {
-                    break;
+                if (!window->isClient() || window->opacity() != 1.0 || !window->isFullScreen()) {
+                    return {};
                 }
                 if (!windowItem->surfaceItem()) {
-                    break;
+                    continue;
                 }
-                SurfaceItem *topMost = findTopMostSurface(windowItem->surfaceItem());
-                if (!topMost) {
-                    break;
+                if (!addCandidates(windowItem->surfaceItem(), ret, maxCount)) {
+                    return {};
                 }
-                // the subsurface has to be able to cover the whole window
-                if (topMost->position() != QPoint(0, 0)) {
-                    break;
-                }
-                // and it has to be completely opaque
-                if (!topMost->opaque().contains(QRect(0, 0, window->width(), window->height()))) {
-                    break;
-                }
-                candidate = topMost;
-                break;
+                return ret;
             }
         }
     }
-    return candidate;
+    return ret;
 }
 
 void WorkspaceScene::frame(SceneDelegate *delegate, OutputFrame *frame)

src/scene/workspacescene.h

@@ -49,7 +49,7 @@ public:
     Item *containerItem() const;
     Item *overlayItem() const;
 
-    SurfaceItem *scanoutCandidate() const override;
+    QList<SurfaceItem *> scanoutCandidates(ssize_t maxCount) const override;
     QRegion prePaint(SceneDelegate *delegate) override;
     void postPaint() override;
     void paint(const RenderTarget &renderTarget, const QRegion &region) override;