diff --git a/src/useractions.cpp b/src/useractions.cpp
index 37794f4e8d..ca738818ea 100644
--- a/src/useractions.cpp
+++ b/src/useractions.cpp
@@ -1131,6 +1131,11 @@ void Workspace::windowShortcutUpdated(Window *window)
     if (!window->shortcut().isEmpty()) {
         if (action == nullptr) { // new shortcut
             action = new QAction(this);
+            connect(window, &Window::closed, action, [action]() {
+                KGlobalAccel::self()->removeAllShortcuts(action);
+                delete action;
+            });
+
             action->setProperty("componentName", QStringLiteral("kwin"));
             action->setObjectName(key);
             action->setText(i18n("Activate Window (%1)", window->caption()));
diff --git a/src/workspace.cpp b/src/workspace.cpp
index 03782a6f61..03734f2204 100644
--- a/src/workspace.cpp
+++ b/src/workspace.cpp
@@ -836,10 +836,6 @@ void Workspace::removeWindow(Window *window)
     if (m_windowKeysWindow == window) {
         setupWindowShortcutDone(false);
     }
-    if (!window->shortcut().isEmpty()) {
-        window->setShortcut(QString()); // Remove from client_keys
-        windowShortcutUpdated(window); // Needed, since this is otherwise delayed by setShortcut() and wouldn't run
-    }
     if (window->hasStrut()) {
         updateClientArea();
     }