From def0bde5e90bd89a783e2d402df619dc7a9dc13a Mon Sep 17 00:00:00 2001
From: Xaver Hugl <xaver.hugl@gmail.com>
Date: Fri, 2 Aug 2024 20:46:39 +0200
Subject: [PATCH] scene/workspacescene: do occlusion testing for direct scanout

Some clients have two or more completely opaque surfaces stacked on top of each other,
optimizing the lower ones out makes direct scanout happen more often and more efficiently
when multiple planes are involved
---
 src/scene/workspacescene.cpp | 20 ++++++++++++++------
 1 file changed, 14 insertions(+), 6 deletions(-)

diff --git a/src/scene/workspacescene.cpp b/src/scene/workspacescene.cpp
index f6affdcea1..ad7742b003 100644
--- a/src/scene/workspacescene.cpp
+++ b/src/scene/workspacescene.cpp
@@ -151,12 +151,15 @@ Item *WorkspaceScene::overlayItem() const
     return m_overlayItem.get();
 }
 
-static bool addCandidates(SurfaceItem *item, QList<SurfaceItem *> &candidates, ssize_t maxCount)
+static bool addCandidates(SurfaceItem *item, QList<SurfaceItem *> &candidates, ssize_t maxCount, QRegion &occluded)
 {
     const QList<Item *> children = item->sortedChildItems();
     for (const auto &child : children | std::views::reverse) {
-        if (child->z() >= 0 && child->isVisible()) {
-            if (!addCandidates(static_cast<SurfaceItem *>(child), candidates, maxCount)) {
+        if (child->z() < 0) {
+            break;
+        }
+        if (child->isVisible() && !occluded.contains(child->mapToScene(child->boundingRect()).toAlignedRect())) {
+            if (!addCandidates(static_cast<SurfaceItem *>(child), candidates, maxCount, occluded)) {
                 return false;
             }
         }
@@ -165,9 +168,13 @@ static bool addCandidates(SurfaceItem *item, QList<SurfaceItem *> &candidates, s
         return false;
     }
     candidates.push_back(item);
+    occluded += item->mapToScene(item->opaque());
     for (const auto &child : children | std::views::reverse) {
-        if (child->z() < 0 && child->isVisible()) {
-            if (!addCandidates(static_cast<SurfaceItem *>(child), candidates, maxCount)) {
+        if (child->z() >= 0) {
+            continue;
+        }
+        if (child->isVisible() && !occluded.contains(child->mapToScene(child->boundingRect()).toAlignedRect())) {
+            if (!addCandidates(static_cast<SurfaceItem *>(child), candidates, maxCount, occluded)) {
                 return false;
             }
         }
@@ -192,7 +199,8 @@ QList<SurfaceItem *> WorkspaceScene::scanoutCandidates(ssize_t maxCount) const
                 if (!windowItem->surfaceItem()) {
                     continue;
                 }
-                if (!addCandidates(windowItem->surfaceItem(), ret, maxCount)) {
+                QRegion occlusion;
+                if (!addCandidates(windowItem->surfaceItem(), ret, maxCount, occlusion)) {
                     return {};
                 }
                 return ret;