QWidget::childAt for masked child widgets doesn't work properly

Problem was that we didn't take the children's effective mask
into account when checking whether a point was inside or not.

This commit is also an optimization since we no longer check the point
against the widget's rect twice. Furthermore, unnecessary
QRegion operations are completely avoided in the common case.

Auto tests included.

Task-number: QTBUG-7150
Reviewed-by: paul
(cherry picked from commit 7101ae5a0e90d97acf86a444c4d51ca45e7863fe)

2 files changed, 53 insertions, 31 deletions

diff --git a/src/gui/kernel/qwidget.cpp b/src/gui/kernel/qwidget.cpp
index 88dcac5..e73f98a 100644
--- a/src/gui/kernel/qwidget.cpp
+++ b/src/gui/kernel/qwidget.cpp
@@ -67,6 +67,7 @@
 # include "qt_mac_p.h"
 # include "qt_cocoa_helpers_mac_p.h"
 # include "qmainwindow.h"
+# include "qtoolbar.h"
 #endif
 #if defined(Q_WS_QWS)
 # include "qwsdisplay_qws.h"
@@ -9606,46 +9607,58 @@ QWidget *QWidget::childAt(const QPoint &p) const
 
 QWidget *QWidgetPrivate::childAt_helper(const QPoint &p, bool ignoreChildrenInDestructor) const
 {
-    Q_Q(const QWidget);
+    if (children.isEmpty())
+        return 0;
+
 #ifdef Q_WS_MAC
+    Q_Q(const QWidget);
+    // Unified tool bars on the Mac require special handling since they live outside
+    // QMainWindow's geometry(). See commit: 35667fd45ada49269a5987c235fdedfc43e92bb8
     bool includeFrame = q->isWindow() && qobject_cast<const QMainWindow *>(q)
                         && static_cast<const QMainWindow *>(q)->unifiedTitleAndToolBarOnMac();
+    if (includeFrame)
+        return childAtRecursiveHelper(p, ignoreChildrenInDestructor, includeFrame);
 #endif
 
-    if (
-#ifdef Q_WS_MAC
-            !includeFrame &&
-#endif
-            !q->rect().contains(p))
+    if (!pointInsideRectAndMask(p))
         return 0;
+    return childAtRecursiveHelper(p, ignoreChildrenInDestructor);
+}
 
-    for (int i = children.size(); i > 0 ;) {
-        --i;
-        QWidget *w = qobject_cast<QWidget *>(children.at(i));
-        if (w && !w->isWindow() && !w->isHidden()
-                && (w->geometry().contains(p)
-#ifdef Q_WS_MAC
-                    || (includeFrame && w->geometry().contains(qt_mac_nativeMapFromParent(w, p)))
+QWidget *QWidgetPrivate::childAtRecursiveHelper(const QPoint &p, bool ignoreChildrenInDestructor, bool includeFrame) const
+{
+#ifndef Q_WS_MAC
+    Q_UNUSED(includeFrame);
 #endif
-               )) {
-            if (ignoreChildrenInDestructor && w->data->in_destructor)
-                continue;
-            if (w->testAttribute(Qt::WA_TransparentForMouseEvents))
-                continue;
-            QPoint childPoint = w->mapFromParent(p);
-#ifdef Q_WS_MAC
-            if (includeFrame && !w->geometry().contains(p))
-                childPoint = qt_mac_nativeMapFromParent(w, p);
-#endif
-            if (QWidget *t = w->d_func()->childAt_helper(childPoint, ignoreChildrenInDestructor))
-                return t;
-            // if WMouseNoMask is set the widget mask is ignored, if
-            // the widget has no mask then the WMouseNoMask flag has no
-            // effect
-            if (w->testAttribute(Qt::WA_MouseNoMask) || w->mask().contains(childPoint)
-                || w->mask().isEmpty())
-                return w;
+    for (int i = children.size() - 1; i >= 0; --i) {
+        QWidget *child = qobject_cast<QWidget *>(children.at(i));
+        if (!child || child->isWindow() || child->isHidden() || child->testAttribute(Qt::WA_TransparentForMouseEvents)
+            || (ignoreChildrenInDestructor && child->data->in_destructor)) {
+            continue;
         }
+
+        // Map the point 'p' from parent coordinates to child coordinates.
+        QPoint childPoint = p;
+#ifdef Q_WS_MAC
+        // 'includeFrame' is true if the child's parent is a top-level QMainWindow with an unified tool bar.
+        // An unified tool bar on the Mac lives outside QMainWindow's geometry(), so a normal
+        // QWidget::mapFromParent won't do the trick.
+        if (includeFrame && qobject_cast<QToolBar *>(child))
+            childPoint = qt_mac_nativeMapFromParent(child, p);
+        else
+#endif
+        childPoint -= child->data->crect.topLeft();
+
+        // Check if the point hits the child.
+        if (!child->d_func()->pointInsideRectAndMask(childPoint))
+            continue;
+
+        // Do the same for the child's descendants.
+        if (QWidget *w = child->d_func()->childAtRecursiveHelper(childPoint, ignoreChildrenInDestructor))
+            return w;
+
+        // We have found our target; namely the child at position 'p'.
+        return child;
     }
     return 0;
 }
diff --git a/src/gui/kernel/qwidget_p.h b/src/gui/kernel/qwidget_p.h
index 9d32a70..aa88112 100644
--- a/src/gui/kernel/qwidget_p.h
+++ b/src/gui/kernel/qwidget_p.h
@@ -491,7 +491,9 @@ public:
     bool setMinimumSize_helper(int &minw, int &minh);
     bool setMaximumSize_helper(int &maxw, int &maxh);
     void setConstraints_sys();
+    bool pointInsideRectAndMask(const QPoint &) const;
     QWidget *childAt_helper(const QPoint &, bool) const;
+    QWidget *childAtRecursiveHelper(const QPoint &p, bool, bool includeFrame = false) const;
     void updateGeometry_helper(bool forceUpdate);
 
     void getLayoutItemMargins(int *left, int *top, int *right, int *bottom) const;
@@ -903,6 +905,13 @@ inline void QWidgetPrivate::setSharedPainter(QPainter *painter)
     x->sharedPainter = painter;
 }
 
+inline bool QWidgetPrivate::pointInsideRectAndMask(const QPoint &p) const
+{
+    Q_Q(const QWidget);
+    return q->rect().contains(p) && (!extra || !extra->hasMask || q->testAttribute(Qt::WA_MouseNoMask)
+                                     || extra->mask.contains(p));
+}
+
 inline QWidgetBackingStore *QWidgetPrivate::maybeBackingStore() const
 {
     Q_Q(const QWidget);