QWidget::childAt for masked child widgets doesn't work properly

Problem was that we didn't take the children's effective mask
into account when checking whether a point was inside or not.

This commit is also an optimization since we no longer check the point
against the widget's rect twice. Furthermore, unnecessary
QRegion operations are completely avoided in the common case.

Auto tests included.

Task-number: QTBUG-7150
Reviewed-by: paul
(cherry picked from commit 7101ae5a0e90d97acf86a444c4d51ca45e7863fe)

3 files changed, 150 insertions, 31 deletions

diff --git a/src/gui/kernel/qwidget.cpp b/src/gui/kernel/qwidget.cpp
index 88dcac5..e73f98a 100644
--- a/src/gui/kernel/qwidget.cpp
+++ b/src/gui/kernel/qwidget.cpp
@@ -67,6 +67,7 @@
 # include "qt_mac_p.h"
 # include "qt_cocoa_helpers_mac_p.h"
 # include "qmainwindow.h"
+# include "qtoolbar.h"
 #endif
 #if defined(Q_WS_QWS)
 # include "qwsdisplay_qws.h"
@@ -9606,46 +9607,58 @@ QWidget *QWidget::childAt(const QPoint &p) const
 
 QWidget *QWidgetPrivate::childAt_helper(const QPoint &p, bool ignoreChildrenInDestructor) const
 {
-    Q_Q(const QWidget);
+    if (children.isEmpty())
+        return 0;
+
 #ifdef Q_WS_MAC
+    Q_Q(const QWidget);
+    // Unified tool bars on the Mac require special handling since they live outside
+    // QMainWindow's geometry(). See commit: 35667fd45ada49269a5987c235fdedfc43e92bb8
     bool includeFrame = q->isWindow() && qobject_cast<const QMainWindow *>(q)
                         && static_cast<const QMainWindow *>(q)->unifiedTitleAndToolBarOnMac();
+    if (includeFrame)
+        return childAtRecursiveHelper(p, ignoreChildrenInDestructor, includeFrame);
 #endif
 
-    if (
-#ifdef Q_WS_MAC
-            !includeFrame &&
-#endif
-            !q->rect().contains(p))
+    if (!pointInsideRectAndMask(p))
         return 0;
+    return childAtRecursiveHelper(p, ignoreChildrenInDestructor);
+}
 
-    for (int i = children.size(); i > 0 ;) {
-        --i;
-        QWidget *w = qobject_cast<QWidget *>(children.at(i));
-        if (w && !w->isWindow() && !w->isHidden()
-                && (w->geometry().contains(p)
-#ifdef Q_WS_MAC
-                    || (includeFrame && w->geometry().contains(qt_mac_nativeMapFromParent(w, p)))
+QWidget *QWidgetPrivate::childAtRecursiveHelper(const QPoint &p, bool ignoreChildrenInDestructor, bool includeFrame) const
+{
+#ifndef Q_WS_MAC
+    Q_UNUSED(includeFrame);
 #endif
-               )) {
-            if (ignoreChildrenInDestructor && w->data->in_destructor)
-                continue;
-            if (w->testAttribute(Qt::WA_TransparentForMouseEvents))
-                continue;
-            QPoint childPoint = w->mapFromParent(p);
-#ifdef Q_WS_MAC
-            if (includeFrame && !w->geometry().contains(p))
-                childPoint = qt_mac_nativeMapFromParent(w, p);
-#endif
-            if (QWidget *t = w->d_func()->childAt_helper(childPoint, ignoreChildrenInDestructor))
-                return t;
-            // if WMouseNoMask is set the widget mask is ignored, if
-            // the widget has no mask then the WMouseNoMask flag has no
-            // effect
-            if (w->testAttribute(Qt::WA_MouseNoMask) || w->mask().contains(childPoint)
-                || w->mask().isEmpty())
-                return w;
+    for (int i = children.size() - 1; i >= 0; --i) {
+        QWidget *child = qobject_cast<QWidget *>(children.at(i));
+        if (!child || child->isWindow() || child->isHidden() || child->testAttribute(Qt::WA_TransparentForMouseEvents)
+            || (ignoreChildrenInDestructor && child->data->in_destructor)) {
+            continue;
         }
+
+        // Map the point 'p' from parent coordinates to child coordinates.
+        QPoint childPoint = p;
+#ifdef Q_WS_MAC
+        // 'includeFrame' is true if the child's parent is a top-level QMainWindow with an unified tool bar.
+        // An unified tool bar on the Mac lives outside QMainWindow's geometry(), so a normal
+        // QWidget::mapFromParent won't do the trick.
+        if (includeFrame && qobject_cast<QToolBar *>(child))
+            childPoint = qt_mac_nativeMapFromParent(child, p);
+        else
+#endif
+        childPoint -= child->data->crect.topLeft();
+
+        // Check if the point hits the child.
+        if (!child->d_func()->pointInsideRectAndMask(childPoint))
+            continue;
+
+        // Do the same for the child's descendants.
+        if (QWidget *w = child->d_func()->childAtRecursiveHelper(childPoint, ignoreChildrenInDestructor))
+            return w;
+
+        // We have found our target; namely the child at position 'p'.
+        return child;
     }
     return 0;
 }
diff --git a/src/gui/kernel/qwidget_p.h b/src/gui/kernel/qwidget_p.h
index 9d32a70..aa88112 100644
--- a/src/gui/kernel/qwidget_p.h
+++ b/src/gui/kernel/qwidget_p.h
@@ -491,7 +491,9 @@ public:
     bool setMinimumSize_helper(int &minw, int &minh);
     bool setMaximumSize_helper(int &maxw, int &maxh);
     void setConstraints_sys();
+    bool pointInsideRectAndMask(const QPoint &) const;
     QWidget *childAt_helper(const QPoint &, bool) const;
+    QWidget *childAtRecursiveHelper(const QPoint &p, bool, bool includeFrame = false) const;
     void updateGeometry_helper(bool forceUpdate);
 
     void getLayoutItemMargins(int *left, int *top, int *right, int *bottom) const;
@@ -903,6 +905,13 @@ inline void QWidgetPrivate::setSharedPainter(QPainter *painter)
     x->sharedPainter = painter;
 }
 
+inline bool QWidgetPrivate::pointInsideRectAndMask(const QPoint &p) const
+{
+    Q_Q(const QWidget);
+    return q->rect().contains(p) && (!extra || !extra->hasMask || q->testAttribute(Qt::WA_MouseNoMask)
+                                     || extra->mask.contains(p));
+}
+
 inline QWidgetBackingStore *QWidgetPrivate::maybeBackingStore() const
 {
     Q_Q(const QWidget);
diff --git a/tests/auto/qwidget/tst_qwidget.cpp b/tests/auto/qwidget/tst_qwidget.cpp
index b3075b9..64db533 100644
--- a/tests/auto/qwidget/tst_qwidget.cpp
+++ b/tests/auto/qwidget/tst_qwidget.cpp
@@ -403,6 +403,10 @@ private slots:
 
     void taskQTBUG_7532_tabOrderWithFocusProxy();
     void movedAndResizedAttributes();
+    void childAt();
+#ifdef Q_WS_MAC
+    void childAt_unifiedToolBar();
+#endif
 
 private:
     bool ensureScreenSize(int width, int height);
@@ -10097,5 +10101,98 @@ void tst_QWidget::movedAndResizedAttributes()
 #endif
 }
 
+void tst_QWidget::childAt()
+{
+    QWidget parent(0, Qt::FramelessWindowHint);
+    parent.resize(200, 200);
+
+    QWidget *child = new QWidget(&parent);
+    child->setPalette(Qt::red);
+    child->setAutoFillBackground(true);
+    child->setGeometry(20, 20, 160, 160);
+
+    QWidget *grandChild = new QWidget(child);
+    grandChild->setPalette(Qt::blue);
+    grandChild->setAutoFillBackground(true);
+    grandChild->setGeometry(-20, -20, 220, 220);
+
+    QVERIFY(!parent.childAt(19, 19));
+    QVERIFY(!parent.childAt(180, 180));
+    QCOMPARE(parent.childAt(20, 20), grandChild);
+    QCOMPARE(parent.childAt(179, 179), grandChild);
+
+    grandChild->setAttribute(Qt::WA_TransparentForMouseEvents);
+    QCOMPARE(parent.childAt(20, 20), child);
+    QCOMPARE(parent.childAt(179, 179), child);
+    grandChild->setAttribute(Qt::WA_TransparentForMouseEvents, false);
+
+    child->setMask(QRect(50, 50, 60, 60));
+
+    QVERIFY(!parent.childAt(69, 69));
+    QVERIFY(!parent.childAt(130, 130));
+    QCOMPARE(parent.childAt(70, 70), grandChild);
+    QCOMPARE(parent.childAt(129, 129), grandChild);
+
+    child->setAttribute(Qt::WA_MouseNoMask);
+    QCOMPARE(parent.childAt(69, 69), grandChild);
+    QCOMPARE(parent.childAt(130, 130), grandChild);
+    child->setAttribute(Qt::WA_MouseNoMask, false);
+
+    grandChild->setAttribute(Qt::WA_TransparentForMouseEvents);
+    QCOMPARE(parent.childAt(70, 70), child);
+    QCOMPARE(parent.childAt(129, 129), child);
+    grandChild->setAttribute(Qt::WA_TransparentForMouseEvents, false);
+
+    grandChild->setMask(QRect(80, 80, 40, 40));
+
+    QCOMPARE(parent.childAt(79, 79), child);
+    QCOMPARE(parent.childAt(120, 120), child);
+    QCOMPARE(parent.childAt(80, 80), grandChild);
+    QCOMPARE(parent.childAt(119, 119), grandChild);
+
+    grandChild->setAttribute(Qt::WA_MouseNoMask);
+
+    QCOMPARE(parent.childAt(79, 79), grandChild);
+    QCOMPARE(parent.childAt(120, 120), grandChild);
+}
+
+#ifdef Q_WS_MAC
+void tst_QWidget::childAt_unifiedToolBar()
+{
+    QLabel *label = new QLabel(QLatin1String("foo"));
+    QToolBar *toolBar = new QToolBar;
+    toolBar->addWidget(new QLabel("dummy"));
+    toolBar->addWidget(label);
+
+    QMainWindow mainWindow;
+    mainWindow.addToolBar(toolBar);
+    mainWindow.show();
+
+    // Calculate the top-left corner of the tool bar and the label (in mainWindow's coordinates).
+    QPoint labelTopLeft = label->mapTo(&mainWindow, QPoint());
+    QPoint toolBarTopLeft = toolBar->mapTo(&mainWindow, QPoint());
+
+    QCOMPARE(mainWindow.childAt(toolBarTopLeft), static_cast<QWidget *>(toolBar));
+    QCOMPARE(mainWindow.childAt(labelTopLeft), static_cast<QWidget *>(label));
+
+    // Enable unified tool bars.
+    mainWindow.setUnifiedTitleAndToolBarOnMac(true);
+    QTest::qWait(50);
+
+    // The tool bar is now in the "non-client" area of QMainWindow, i.e.
+    // outside the mainWindow's rect(), and since mapTo et al. doesn't work
+    // in that case (see commit 35667fd45ada49269a5987c235fdedfc43e92bb8),
+    // we use mapToGlobal/mapFromGlobal to re-calculate the corners.
+    QPoint oldToolBarTopLeft = toolBarTopLeft;
+    toolBarTopLeft = mainWindow.mapFromGlobal(toolBar->mapToGlobal(QPoint()));
+    QVERIFY(toolBarTopLeft != oldToolBarTopLeft);
+    QVERIFY(toolBarTopLeft.y() < 0);
+    labelTopLeft = mainWindow.mapFromGlobal(label->mapToGlobal(QPoint()));
+
+    QCOMPARE(mainWindow.childAt(toolBarTopLeft), static_cast<QWidget *>(toolBar));
+    QCOMPARE(mainWindow.childAt(labelTopLeft), static_cast<QWidget *>(label));
+}
+#endif
+
 QTEST_MAIN(tst_QWidget)
 #include "tst_qwidget.moc"