Add interactive position selection to screenshot screen under cursor

Summary:
A second interactive selection mode gets added to select a position on
the screen. This is handled by the same input event filter as for the
window selection. Just that instead of returning a window, it returns a
QPoint.

This allows to pick a point on the screen which we need to screenshot
the screen under the mouse cursor and in future for color picking.

The screenshot effect provides two new dbus methods to (interactively)
select a screen or fullscreen. This allows spectacle to screenshot the
(full) screen with still having the user in control.

Reviewers: #kwin, #plasma_on_wayland, bgupta

Subscribers: plasma-devel, kwin

Tags: #plasma_on_wayland, #kwin

Differential Revision: https://phabricator.kde.org/D3475

autotests/integration/window_selection_test.cpp

@@ -55,6 +55,8 @@ private Q_SLOTS:
     void testSelectOnWindowKeyboard();
     void testCancelOnWindowPointer();
     void testCancelOnWindowKeyboard();
+
+    void testSelectPointPointer();
 };
 
 void TestWindowSelection::initTestCase()
@@ -365,5 +367,89 @@ void TestWindowSelection::testCancelOnWindowKeyboard()
     kwinApp()->platform()->keyboardKeyReleased(KEY_ESC, timestamp++);
 }
 
+void TestWindowSelection::testSelectPointPointer()
+{
+    // this test verifies point selection through pointer works
+    QScopedPointer<Surface> surface(Test::createSurface());
+    QScopedPointer<ShellSurface> shellSurface(Test::createShellSurface(surface.data()));
+    QScopedPointer<Pointer> pointer(Test::waylandSeat()->createPointer());
+    QScopedPointer<Keyboard> keyboard(Test::waylandSeat()->createKeyboard());
+    QSignalSpy pointerEnteredSpy(pointer.data(), &Pointer::entered);
+    QVERIFY(pointerEnteredSpy.isValid());
+    QSignalSpy pointerLeftSpy(pointer.data(), &Pointer::left);
+    QVERIFY(pointerLeftSpy.isValid());
+    QSignalSpy keyboardEnteredSpy(keyboard.data(), &Keyboard::entered);
+    QVERIFY(keyboardEnteredSpy.isValid());
+    QSignalSpy keyboardLeftSpy(keyboard.data(), &Keyboard::left);
+    QVERIFY(keyboardLeftSpy.isValid());
+
+    auto client = Test::renderAndWaitForShown(surface.data(), QSize(100, 50), Qt::blue);
+    QVERIFY(client);
+    QVERIFY(keyboardEnteredSpy.wait());
+    KWin::Cursor::setPos(client->geometry().center());
+    QCOMPARE(input()->pointer()->window().data(), client);
+    QVERIFY(pointerEnteredSpy.wait());
+
+    QPoint point;
+    auto callback = [&point] (const QPoint &p) {
+        point = p;
+    };
+
+    // start the interaction
+    QCOMPARE(input()->isSelectingWindow(), false);
+    kwinApp()->platform()->startInteractivePositionSelection(callback);
+    QCOMPARE(input()->isSelectingWindow(), true);
+    QCOMPARE(point, QPoint());
+    QCOMPARE(keyboardLeftSpy.count(), 0);
+    QVERIFY(pointerLeftSpy.wait());
+    if (keyboardLeftSpy.isEmpty()) {
+        QVERIFY(keyboardLeftSpy.wait());
+    }
+    QCOMPARE(pointerLeftSpy.count(), 1);
+    QCOMPARE(keyboardLeftSpy.count(), 1);
+
+    // trying again should not be allowed
+    QPoint point2;
+    kwinApp()->platform()->startInteractivePositionSelection([&point2] (const QPoint &p) {
+        point2 = p;
+    });
+    QCOMPARE(point2, QPoint(-1, -1));
+
+    // simulate left button press
+    quint32 timestamp = 0;
+    kwinApp()->platform()->pointerButtonPressed(BTN_LEFT, timestamp++);
+    // should not have ended the mode
+    QCOMPARE(input()->isSelectingWindow(), true);
+    QCOMPARE(point, QPoint());
+    QVERIFY(input()->pointer()->window().isNull());
+
+    // updating the pointer should not change anything
+    input()->pointer()->update();
+    QVERIFY(input()->pointer()->window().isNull());
+    // updating keyboard should also not change
+    input()->keyboard()->update();
+
+    // perform a right button click
+    kwinApp()->platform()->pointerButtonPressed(BTN_RIGHT, timestamp++);
+    kwinApp()->platform()->pointerButtonReleased(BTN_RIGHT, timestamp++);
+    // should not have ended the mode
+    QCOMPARE(input()->isSelectingWindow(), true);
+    QCOMPARE(point, QPoint());
+    // now release
+    kwinApp()->platform()->pointerButtonReleased(BTN_LEFT, timestamp++);
+    QCOMPARE(input()->isSelectingWindow(), false);
+    QCOMPARE(point, input()->globalPointer().toPoint());
+    QCOMPARE(input()->pointer()->window().data(), client);
+    // should give back keyboard and pointer
+    QVERIFY(pointerEnteredSpy.wait());
+    if (keyboardEnteredSpy.count() != 2) {
+        QVERIFY(keyboardEnteredSpy.wait());
+    }
+    QCOMPARE(pointerLeftSpy.count(), 1);
+    QCOMPARE(keyboardLeftSpy.count(), 1);
+    QCOMPARE(pointerEnteredSpy.count(), 2);
+    QCOMPARE(keyboardEnteredSpy.count(), 2);
+}
+
 WAYLANDTEST_MAIN(TestWindowSelection)
 #include "window_selection_test.moc"

autotests/mock_effectshandler.h

@@ -248,6 +248,9 @@ public:
     void startInteractiveWindowSelection(std::function<void(KWin::EffectWindow*)> callback) override {
         callback(nullptr);
     }
+    void startInteractivePositionSelection(std::function<void (const QPoint &)> callback) override {
+        callback(QPoint(-1, -1));
+    }
 
 private:
     bool m_animationsSuported = true;

effects.cpp

@@ -1585,6 +1585,11 @@ void EffectsHandlerImpl::startInteractiveWindowSelection(std::function<void(KWin
     );
 }
 
+void EffectsHandlerImpl::startInteractivePositionSelection(std::function<void(const QPoint&)> callback)
+{
+    kwinApp()->platform()->startInteractivePositionSelection(callback);
+}
+
 //****************************************
 // EffectWindowImpl
 //****************************************

effects.h

@@ -232,6 +232,7 @@ public:
     void showCursor() override;
 
     void startInteractiveWindowSelection(std::function<void(KWin::EffectWindow*)> callback) override;
+    void startInteractivePositionSelection(std::function<void(const QPoint &)> callback) override;
 
     Scene *scene() const {
         return m_scene;

effects/screenshot/screenshot.cpp

@@ -251,7 +251,22 @@ void ScreenShotEffect::postPaintScreen()
 
 void ScreenShotEffect::sendReplyImage(const QImage &img)
 {
-    m_replyConnection.send(m_replyMessage.createReply(saveTempImage(img)));
+    if (m_fd != -1) {
+        QtConcurrent::run(
+            [] (int fd, const QImage &img) {
+                QFile file;
+                if (file.open(fd, QIODevice::WriteOnly, QFileDevice::AutoCloseHandle)) {
+                    QDataStream ds(&file);
+                    ds << img;
+                    file.close();
+                } else {
+                    close(fd);
+                }
+            }, m_fd, img);
+        m_fd = -1;
+    } else {
+        m_replyConnection.send(m_replyMessage.createReply(saveTempImage(img)));
+    }
     m_scheduledGeometry = QRect();
     m_multipleOutputsImage = QImage();
     m_multipleOutputsRendered = QRegion();
@@ -335,7 +350,7 @@ QString ScreenShotEffect::interactive(int mask)
             }
     });
 
-    showInfoMessage();
+    showInfoMessage(InfoMessageMode::Window);
     return QString();
 }
 
@@ -370,10 +385,10 @@ void ScreenShotEffect::interactive(QDBusUnixFileDescriptor fd, int mask)
             }
     });
 
-    showInfoMessage();
+    showInfoMessage(InfoMessageMode::Window);
 }
 
-void ScreenShotEffect::showInfoMessage()
+void ScreenShotEffect::showInfoMessage(InfoMessageMode mode)
 {
     if (!m_infoFrame.isNull()) {
         return;
@@ -384,7 +399,14 @@ void ScreenShotEffect::showInfoMessage()
     m_infoFrame->setFont(font);
     QRect area = effects->clientArea(ScreenArea, effects->activeScreen(), effects->currentDesktop());
     m_infoFrame->setPosition(QPoint(area.x() + area.width() / 2, area.y() + area.height() / 3));
-    m_infoFrame->setText(i18n("Select window to screen shot with left click or enter.\nEscape or right click to cancel."));
+    switch (mode) {
+    case InfoMessageMode::Window:
+        m_infoFrame->setText(i18n("Select window to screen shot with left click or enter.\nEscape or right click to cancel."));
+        break;
+    case InfoMessageMode::Screen:
+        m_infoFrame->setText(i18n("Create screen shot with left click or enter.\nEscape or right click to cancel."));
+        break;
+    }
     effects->addRepaintFull();
 }
 
@@ -411,6 +433,38 @@ QString ScreenShotEffect::screenshotFullscreen(bool captureCursor)
     return QString();
 }
 
+void ScreenShotEffect::screenshotFullscreen(QDBusUnixFileDescriptor fd, bool captureCursor)
+{
+    if (!calledFromDBus()) {
+        return;
+    }
+    if (!m_scheduledGeometry.isNull()) {
+        sendErrorReply(QDBusError::Failed, "A screenshot is already been taken");
+        return;
+    }
+    m_fd = dup(fd.fileDescriptor());
+    if (m_fd == -1) {
+        sendErrorReply(QDBusError::Failed, "No valid file descriptor");
+        return;
+    }
+    m_captureCursor = captureCursor;
+
+    showInfoMessage(InfoMessageMode::Screen);
+    effects->startInteractivePositionSelection(
+        [this] (const QPoint &p) {
+            hideInfoMessage();
+            if (p == QPoint(-1, -1)) {
+                // error condition
+                close(m_fd);
+                m_fd = -1;
+            } else {
+                m_scheduledGeometry = effects->virtualScreenGeometry();
+                effects->addRepaint(m_scheduledGeometry);
+            }
+        }
+    );
+}
+
 QString ScreenShotEffect::screenshotScreen(int screen, bool captureCursor)
 {
     if (!calledFromDBus()) {
@@ -433,6 +487,43 @@ QString ScreenShotEffect::screenshotScreen(int screen, bool captureCursor)
     return QString();
 }
 
+void ScreenShotEffect::screenshotScreen(QDBusUnixFileDescriptor fd, bool captureCursor)
+{
+    if (!calledFromDBus()) {
+        return;
+    }
+    if (!m_scheduledGeometry.isNull()) {
+        sendErrorReply(QDBusError::Failed, "A screenshot is already been taken");
+        return;
+    }
+    m_fd = dup(fd.fileDescriptor());
+    if (m_fd == -1) {
+        sendErrorReply(QDBusError::Failed, "No valid file descriptor");
+        return;
+    }
+    m_captureCursor = captureCursor;
+
+    showInfoMessage(InfoMessageMode::Screen);
+    effects->startInteractivePositionSelection(
+        [this] (const QPoint &p) {
+            hideInfoMessage();
+            if (p == QPoint(-1, -1)) {
+                // error condition
+                close(m_fd);
+                m_fd = -1;
+            } else {
+                m_scheduledGeometry = effects->clientArea(FullScreenArea, effects->screenNumber(p), 0);
+                if (m_scheduledGeometry.isNull()) {
+                    close(m_fd);
+                    m_fd = -1;
+                    return;
+                }
+                effects->addRepaint(m_scheduledGeometry);
+            }
+        }
+    );
+}
+
 QString ScreenShotEffect::screenshotArea(int x, int y, int width, int height, bool captureCursor)
 {
     if (!calledFromDBus()) {

effects/screenshot/screenshot.h

@@ -85,6 +85,20 @@ public Q_SLOTS:
      * @returns Path to stored screenshot, or null string in failure case.
      **/
     Q_SCRIPTABLE QString screenshotFullscreen(bool captureCursor = false);
+    /**
+     * Starts an interactive screenshot session.
+     *
+     * The user is asked to confirm that a screenshot is taken by having to actively
+     * click and giving the possibility to cancel.
+     *
+     * Once the screenshot is taken it gets saved into the @p fd passed to the
+     * method. It is intended to be used with a pipe, so that the invoking side can just
+     * read from the pipe. The image gets written into the fd using a QDataStream.
+     *
+     * @param fd File descriptor into which the screenshot should be saved
+     * @param captureCursor Whether to include the mouse cursor
+     **/
+    Q_SCRIPTABLE void screenshotFullscreen(QDBusUnixFileDescriptor fd, bool captureCursor = false);
     /**
      * Saves a screenshot of the screen identified by @p screen into a file and returns the path to the file.
      * Functionality requires hardware support, if not available a null string is returned.
@@ -93,6 +107,19 @@ public Q_SLOTS:
      * @returns Path to stored screenshot, or null string in failure case.
      **/
     Q_SCRIPTABLE QString screenshotScreen(int screen, bool captureCursor = false);
+    /**
+     * Starts an interactive screenshot of a screen session.
+     *
+     * The user is asked to select the screen to screenshot.
+     *
+     * Once the screenshot is taken it gets saved into the @p fd passed to the
+     * method. It is intended to be used with a pipe, so that the invoking side can just
+     * read from the pipe. The image gets written into the fd using a QDataStream.
+     *
+     * @param fd File descriptor into which the screenshot should be saved
+     * @param captureCursor Whether to include the mouse cursor
+     **/
+    Q_SCRIPTABLE void screenshotScreen(QDBusUnixFileDescriptor fd, bool captureCursor = false);
     /**
      * Saves a screenshot of the selected geometry into a file and returns the path to the file.
      * Functionality requires hardware support, if not available a null string is returned.
@@ -116,7 +143,11 @@ private:
     QImage blitScreenshot(const QRect &geometry);
     QString saveTempImage(const QImage &img);
     void sendReplyImage(const QImage &img);
-    void showInfoMessage();
+    enum class InfoMessageMode {
+        Window,
+        Screen
+    };
+    void showInfoMessage(InfoMessageMode mode);
     void hideInfoMessage();
     EffectWindow *m_scheduledScreenshot;
     ScreenShotType m_type;

input.cpp

@@ -554,24 +554,42 @@ public:
         m_callback = callback;
         input()->keyboard()->update();
     }
+    void start(std::function<void(const QPoint &)> callback) {
+        Q_ASSERT(!m_active);
+        m_active = true;
+        m_pointSelectionFallback = callback;
+        input()->keyboard()->update();
+    }
 private:
     void deactivate() {
         m_active = false;
         m_callback = std::function<void(KWin::Toplevel*)>();
+        m_pointSelectionFallback = std::function<void(const QPoint &)>();
         input()->pointer()->removeWindowSelectionCursor();
         input()->keyboard()->update();
     }
     void cancel() {
-        m_callback(nullptr);
+        if (m_callback) {
+            m_callback(nullptr);
+        }
+        if (m_pointSelectionFallback) {
+            m_pointSelectionFallback(QPoint(-1, -1));
+        }
         deactivate();
     }
     void accept() {
-        // TODO: this ignores shaped windows
-        m_callback(input()->findToplevel(input()->globalPointer().toPoint()));
+        if (m_callback) {
+            // TODO: this ignores shaped windows
+            m_callback(input()->findToplevel(input()->globalPointer().toPoint()));
+        }
+        if (m_pointSelectionFallback) {
+            m_pointSelectionFallback(input()->globalPointer().toPoint());
+        }
         deactivate();
     }
     bool m_active = false;
     std::function<void(KWin::Toplevel*)> m_callback;
+    std::function<void(const QPoint &)> m_pointSelectionFallback;
 };
 
 class GlobalShortcutFilter : public InputEventFilter {
@@ -1757,6 +1775,16 @@ void InputRedirection::startInteractiveWindowSelection(std::function<void(KWin::
     m_pointer->setWindowSelectionCursor(cursorName);
 }
 
+void InputRedirection::startInteractivePositionSelection(std::function<void(const QPoint &)> callback)
+{
+    if (!m_windowSelector || m_windowSelector->isActive()) {
+        callback(QPoint(-1, -1));
+        return;
+    }
+    m_windowSelector->start(callback);
+    m_pointer->setWindowSelectionCursor(QByteArray());
+}
+
 bool InputRedirection::isSelectingWindow() const
 {
     return m_windowSelector ? m_windowSelector->isActive() : false;

input.h

@@ -170,6 +170,7 @@ public:
     bool hasAlphaNumericKeyboard();
 
     void startInteractiveWindowSelection(std::function<void(KWin::Toplevel*)> callback, const QByteArray &cursorName);
+    void startInteractivePositionSelection(std::function<void(const QPoint &)> callback);
     bool isSelectingWindow() const;
 
 Q_SIGNALS:

libkwineffects/kwineffects.h

@@ -1214,6 +1214,21 @@ public:
      **/
     virtual void startInteractiveWindowSelection(std::function<void(KWin::EffectWindow*)> callback) = 0;
 
+    /**
+     * Starts an interactive position selection process.
+     *
+     * Once the user selected a position on the screen the @p callback is invoked with
+     * the selected point as argument. In case the user cancels the interactive position selection
+     * or selecting a position is currently not possible (e.g. screen locked) the @p callback
+     * is invoked with a point at @c -1 as x and y argument.
+     *
+     * During the interactive window selection the cursor is turned into a crosshair cursor.
+     *
+     * @param callback The function to invoke once the interactive position selection ends
+     * @since 5.9
+     **/
+    virtual void startInteractivePositionSelection(std::function<void(const QPoint &)> callback) = 0;
+
 Q_SIGNALS:
     /**
      * Signal emitted when the current desktop changed.

platform.cpp

@@ -377,4 +377,13 @@ void Platform::startInteractiveWindowSelection(std::function<void(KWin::Toplevel
     input()->startInteractiveWindowSelection(callback, cursorName);
 }
 
+void Platform::startInteractivePositionSelection(std::function<void(const QPoint &)> callback)
+{
+    if (!input()) {
+        callback(QPoint(-1, -1));
+        return;
+    }
+    input()->startInteractivePositionSelection(callback);
+}
+
 }

platform.h

@@ -179,6 +179,22 @@ public:
      **/
     virtual void startInteractiveWindowSelection(std::function<void(KWin::Toplevel*)> callback, const QByteArray &cursorName = QByteArray());
 
+    /**
+     * Starts an interactive position selection process.
+     *
+     * Once the user selected a position on the screen the @p callback is invoked with
+     * the selected point as argument. In case the user cancels the interactive position selection
+     * or selecting a position is currently not possible (e.g. screen locked) the @p callback
+     * is invoked with a point at @c -1 as x and y argument.
+     *
+     * During the interactive window selection the cursor is turned into a crosshair cursor.
+     *
+     * The default implementation forwards to InputRedirection.
+     *
+     * @param callback The function to invoke once the interactive position selection ends
+     **/
+    virtual void startInteractivePositionSelection(std::function<void(const QPoint &)> callback);
+
     bool usesSoftwareCursor() const {
         return m_softWareCursor;
     }