diff --git a/src/surfaceitem_x11.cpp b/src/surfaceitem_x11.cpp
index 4ffc775702..26a10d5022 100644
--- a/src/surfaceitem_x11.cpp
+++ b/src/surfaceitem_x11.cpp
@@ -19,8 +19,6 @@ SurfaceItemX11::SurfaceItemX11(Scene::Window *window, Item *parent)
 
     connect(toplevel, &Toplevel::bufferGeometryChanged,
             this, &SurfaceItemX11::handleBufferGeometryChanged);
-    connect(toplevel, &Toplevel::markedAsZombie,
-            this, &SurfaceItemX11::destroyDamage);
     connect(toplevel, &Toplevel::geometryShapeChanged,
             this, &SurfaceItemX11::discardQuads);
 
@@ -33,6 +31,7 @@ SurfaceItemX11::SurfaceItemX11(Scene::Window *window, Item *parent)
 
 SurfaceItemX11::~SurfaceItemX11()
 {
+    // destroyDamage() will be called by the associated Toplevel.
 }
 
 void SurfaceItemX11::preprocess()
diff --git a/src/toplevel.cpp b/src/toplevel.cpp
index 7ada48d145..bc8b09dc90 100644
--- a/src/toplevel.cpp
+++ b/src/toplevel.cpp
@@ -20,6 +20,7 @@
 #include "screens.h"
 #include "shadow.h"
 #include "shadowitem.h"
+#include "surfaceitem_x11.h"
 #include "windowitem.h"
 #include "workspace.h"
 
@@ -276,8 +277,14 @@ bool Toplevel::setupCompositing()
     return true;
 }
 
-void Toplevel::finishCompositing(ReleaseReason)
+void Toplevel::finishCompositing(ReleaseReason releaseReason)
 {
+    // If the X11 window has been destroyed, avoid calling XDamageDestroy.
+    if (releaseReason != ReleaseReason::Destroyed) {
+        if (SurfaceItemX11 *item = qobject_cast<SurfaceItemX11 *>(surfaceItem())) {
+            item->destroyDamage();
+        }
+    }
     if (effect_window && effect_window->window() == this) { // otherwise it's already passed to Deleted, don't free data
         deleteEffectWindow();
     }