diff --git a/src/xdgshellwindow.cpp b/src/xdgshellwindow.cpp
index 15f42f470a..ce672a6c6e 100644
--- a/src/xdgshellwindow.cpp
+++ b/src/xdgshellwindow.cpp
@@ -1126,8 +1126,9 @@ void XdgToplevelWindow::handlePingDelayed(quint32 serial)
 
 void XdgToplevelWindow::handlePongReceived(quint32 serial)
 {
-    m_pings.remove(serial);
-    setUnresponsive(false);
+    if (m_pings.remove(serial)) {
+        setUnresponsive(false);
+    }
 }
 
 void XdgToplevelWindow::handleMaximumSizeChanged()