From 9c2035ca635e48ce7fc2627f2fb40e7bd27a7ad7 Mon Sep 17 00:00:00 2001
From: Xaver Hugl <xaver.hugl@gmail.com>
Date: Thu, 4 Apr 2024 15:32:11 +0200
Subject: [PATCH] scene/workspacescene: fix direct scanout checks with
 subsurfaces

The check ignored that subsurfaces could be not visible, not mapped, and also below
the parent surface
---
 src/scene/workspacescene.cpp | 31 +++++++++++++++++++++++++------
 1 file changed, 25 insertions(+), 6 deletions(-)

diff --git a/src/scene/workspacescene.cpp b/src/scene/workspacescene.cpp
index 0e107c47f4..d099a7321d 100644
--- a/src/scene/workspacescene.cpp
+++ b/src/scene/workspacescene.cpp
@@ -153,12 +153,28 @@ Item *WorkspaceScene::overlayItem() const
 
 static SurfaceItem *findTopMostSurface(SurfaceItem *item)
 {
-    const QList<Item *> children = item->childItems();
-    if (children.isEmpty()) {
-        return item;
-    } else {
-        return findTopMostSurface(static_cast<SurfaceItem *>(children.constLast()));
+    if (!item->isVisible()) {
+        return nullptr;
     }
+    const QList<Item *> children = item->sortedChildItems();
+    for (const auto &child : children | std::views::reverse) {
+        if (child->z() >= 0) {
+            if (auto item = findTopMostSurface(static_cast<SurfaceItem *>(child))) {
+                return item;
+            }
+        }
+    }
+    if (item->pixmap()) {
+        return item;
+    }
+    for (const auto &child : children | std::views::reverse) {
+        if (child->z() < 0) {
+            if (auto item = findTopMostSurface(static_cast<SurfaceItem *>(child))) {
+                return item;
+            }
+        }
+    }
+    return nullptr;
 }
 
 SurfaceItem *WorkspaceScene::scanoutCandidate() const
@@ -171,7 +187,7 @@ SurfaceItem *WorkspaceScene::scanoutCandidate() const
         for (int i = stacking_order.count() - 1; i >= 0; i--) {
             WindowItem *windowItem = stacking_order[i];
             Window *window = windowItem->window();
-            if (window->isOnOutput(painted_screen) && window->opacity() > 0) {
+            if (window->isOnOutput(painted_screen) && window->opacity() > 0 && windowItem->isVisible()) {
                 if (!window->isClient() || !window->isFullScreen() || window->opacity() != 1.0) {
                     break;
                 }
@@ -179,6 +195,9 @@ SurfaceItem *WorkspaceScene::scanoutCandidate() const
                     break;
                 }
                 SurfaceItem *topMost = findTopMostSurface(windowItem->surfaceItem());
+                if (!topMost) {
+                    break;
+                }
                 // the subsurface has to be able to cover the whole window
                 if (topMost->position() != QPoint(0, 0)) {
                     break;