diff --git a/src/plugins/idletime/poller.cpp b/src/plugins/idletime/poller.cpp
index 6613ad75d9..3ac43e1860 100644
--- a/src/plugins/idletime/poller.cpp
+++ b/src/plugins/idletime/poller.cpp
@@ -39,7 +39,9 @@ bool KWinIdleTimePoller::setUpPoller()
 
 void KWinIdleTimePoller::unloadPoller()
 {
-    disconnect(waylandServer()->idle(), &KWaylandServer::IdleInterface::inhibitedChanged, this, &KWinIdleTimePoller::onInhibitedChanged);
+    if (waylandServer() && waylandServer()->idle()) {
+        disconnect(waylandServer()->idle(), &KWaylandServer::IdleInterface::inhibitedChanged, this, &KWinIdleTimePoller::onInhibitedChanged);
+    }
 
     qDeleteAll(m_timeouts);
     m_timeouts.clear();