Implement sanity checks when placing transients

A transient window should always be visible on the current screen.
This change ensures that a transient is always placed in a way that
the transient window is visible on the screen ignoring the transient
offset hint if it has to be.

Unfortunately QtWayland doesn't set the transient hint correctly:
a sub menu is a transient to the main window and not to the parent
menu resulting in quite off positioned menus, see:
https://bugreports.qt.io/browse/QTBUG-51640

autotests/wayland/transient_placement.cpp

@@ -245,12 +245,20 @@ void TransientPlacementTest::testSimplePosition_data()
     QTest::newRow("0/0") << QSize(640, 512) << QPoint(0, 0) << QSize(10, 100) << QPoint(0, 0) << QRect(0, 0, 10, 100);
     QTest::newRow("bottomRight") << QSize(640, 512) << QPoint(0, 0) << QSize(10, 100) << QPoint(639, 511) << QRect(639, 511, 10, 100);
     QTest::newRow("offset") << QSize(640, 512) << QPoint(200, 300) << QSize(100, 10) << QPoint(320, 256) << QRect(520, 556, 100, 10);
+    QTest::newRow("right border") << QSize(1280, 1024) << QPoint(0, 0) << QSize(10, 100) << QPoint(1279, 50) << QRect(1269, 50, 10, 100);
+    QTest::newRow("bottom border") << QSize(1280, 1024) << QPoint(0, 0) << QSize(10, 100) << QPoint(512, 1020) << QRect(512, 920, 10, 100);
+    QTest::newRow("bottom right") << QSize(1280, 1024) << QPoint(0, 0) << QSize(10, 100) << QPoint(1279, 1020) << QRect(1269, 920, 10, 100);
+    QTest::newRow("top border") << QSize(1280, 1024) << QPoint(0, -100) << QSize(10, 100) << QPoint(512, 50) << QRect(512, 0, 10, 100);
+    QTest::newRow("left border") << QSize(1280, 1024) << QPoint(-100, 0) << QSize(100, 10) << QPoint(50, 512) << QRect(0, 512, 100, 10);
+    QTest::newRow("top left") << QSize(1280, 1024) << QPoint(-100, -100) << QSize(100, 100) << QPoint(50, 50) << QRect(0, 0, 100, 100);
+    QTest::newRow("bottom left") << QSize(1280, 1024) << QPoint(-100, 0) << QSize(100, 100) << QPoint(50, 1000) << QRect(0, 900, 100, 100);
 }
 
 void TransientPlacementTest::testSimplePosition()
 {
     // this test verifies that the position of a transient window is taken from the passed position
     // there are no further constraints like window too large to fit screen, cascading transients, etc
+    // some test cases also verify that the transient fits on the screen
     QFETCH(QSize, parentSize);
     AbstractClient *parent = showWindow(parentSize);
     QVERIFY(parent->clientPos().isNull());

placement.cpp

@@ -496,8 +496,39 @@ void Placement::placeOnScreenDisplay(AbstractClient* c, QRect& area)
 
 void Placement::placeTransient(AbstractClient *c)
 {
-    // TODO: apply sanity checks?
+    const QPoint target = c->transientFor()->pos() + c->transientFor()->clientPos() + c->transientPlacementHint();
-    c->move(c->transientFor()->pos() + c->transientFor()->clientPos() + c->transientPlacementHint());
+    c->move(target);
+    const QRect screen = screens()->geometry(c->transientFor()->screen());
+    // TODO: work around Qt's transient placement of sub-menus, see https://bugreports.qt.io/browse/QTBUG-51640
+#define CHECK \
+    if (screen.contains(c->geometry())) { \
+        return; \
+    }
+    CHECK
+    if (screen.x() + screen.width() < c->x() + c->width()) {
+        // overlaps on right
+        c->move(c->x() - c->width(), c->y());
+        CHECK
+    }
+    if (screen.y() + screen.height() < c->y() + c->height()) {
+        // overlaps on bottom
+        c->move(c->x(), c->y() - c->height());
+        CHECK
+    }
+    if (screen.y() > c->y()) {
+        // top is not on screen
+        c->move(c->x(), screen.y());
+        CHECK
+    }
+    if (screen.x() > c->x()) {
+        // left is not on screen
+        c->move(screen.y(), c->y());
+        CHECK
+    }
+#undef CHECK
+    // so far the sanitizing didn't help, let's move back to orig target position and use keepInArea
+    c->move(target);
+    c->keepInArea(screen);
 }
 
 void Placement::placeDialog(AbstractClient* c, QRect& area, Policy nextPlacement)