Cocoa: p1 bug fix: revert use of subWindowStacking

This reverts commit 3c2373d7ea9bc91bb537c0725984d19ad0fbab01.
After finding yet another bug related to cocoa child windows
(QTBUG-17738), we have no other option than to admit it was a wrong
move to use the API in the first place. Had we only known how many
side-effects and hidden bugs it would introduce. The original problem
we tried to solve were the cases where a stays-on-top parent window
executed a modal child dialog. This child should always stay on top of
its parent, but Cocoa would insist on pushing the window down to the modal
window level upon activating/deactivating the application. Some window
systems will always stack a window child on top of the parent, while
others (X11) seems to be more selective on this issue. On Mac, we already
stack windows a  bit differently, thinking first and foremost on tool
windows.

Since this change is going into a patch release (which is debatable,
since this changes behaviour, but p1 is a p1), we choose to add in a
backdoor for those users who by chance depend on this behaviour. Setting
the env var QT_MAC_USE_CHILDWINDOWS=1 will give you the old code
path, but we plan to remove this for Qt-4.8.

Also, this patch does fix the original bug described above by overriding
the setLevel method in NSWindow, and refuse Cocoa to level down
stays-on-top modal windows.

2 files changed, 29 insertions, 15 deletions

diff --git a/src/gui/kernel/qcocoasharedwindowmethods_mac_p.h b/src/gui/kernel/qcocoasharedwindowmethods_mac_p.h
index 6254061..406e6d4 100644
--- a/src/gui/kernel/qcocoasharedwindowmethods_mac_p.h
+++ b/src/gui/kernel/qcocoasharedwindowmethods_mac_p.h
@@ -157,6 +157,20 @@ QT_END_NAMESPACE
     [NSApp terminate:sender];
 }
 
+- (void)setLevel:(NSInteger)windowLevel
+{
+    // Cocoa will upon activating/deactivating applications level modal
+    // windows up and down, regardsless of any explicit set window level.
+    // To ensure that modal stays-on-top dialogs actually stays on top after
+    // the application is activated (and therefore stacks in front of
+    // other stays-on-top windows), we need to add this little special-case override:
+    QWidget *widget = [[QT_MANGLE_NAMESPACE(QCocoaWindowDelegate) sharedDelegate] qt_qwidgetForWindow:self];
+    if (widget && widget->isModal() && (widget->windowFlags() & Qt::WindowStaysOnTopHint))
+        [super setLevel:NSPopUpMenuWindowLevel];
+    else
+        [super setLevel:windowLevel];
+}
+
 - (void)sendEvent:(NSEvent *)event
 {
     if ([event type] == NSApplicationDefined) {
diff --git a/src/gui/kernel/qwidget_mac.mm b/src/gui/kernel/qwidget_mac.mm
index 7105d23..0f4abfa 100644
--- a/src/gui/kernel/qwidget_mac.mm
+++ b/src/gui/kernel/qwidget_mac.mm
@@ -2794,14 +2794,18 @@ void QWidgetPrivate::transferChildren()
 #ifdef QT_MAC_USE_COCOA
 void QWidgetPrivate::setSubWindowStacking(bool set)
 {
+    // After hitting too many unforeseen bugs trying to put Qt on top of the cocoa child
+    // window API, we have decided to revert this behaviour. For compability
+    // reasons we leave the code here for applications depending on it:
+    static bool use_behaviour_qt473 = !qgetenv("QT_MAC_USE_CHILDWINDOWS").isEmpty();
+    if (use_behaviour_qt473 == false)
+        return;
+
     // This will set/remove a visual relationship between parent and child on screen.
     // The reason for doing this is to ensure that a child always stacks infront of
-    // its parent (expecially if both windows are modal, and the child blocks input to
-    // the parent while at the same time hiding behind it). Unfortunatly is turns out
-    // that [NSWindow addChildWindow] has
+    // its parent. Unfortunatly is turns out that [NSWindow addChildWindow] has
     // several unwanted side-effects, one of them being the moving of a child when
-    // moving the parent, which we choose to accept (if this is unacceptable, consider
-    // using Qt::WA_MacNoCocoaChildWindow). A way tougher side-effect is
+    // moving the parent, which we choose to accept. A way tougher side-effect is
     // that Cocoa will hide the parent if you hide the child. And in the case of
     // a tool window, since it will normally hide when you deactivate the
     // application, Cocoa will hide the parent upon deactivate as well. The result often
@@ -2824,14 +2828,12 @@ void QWidgetPrivate::setSubWindowStacking(bool set)
     if (QWidget *parent = q->parentWidget()) {
         if (NSWindow *pwin = [qt_mac_nativeview_for(parent) window]) {
             if (set) {
-                if (!q->testAttribute(Qt::WA_MacNoCocoaChildWindow)) {
-                    Qt::WindowType ptype = parent->window()->windowType();
-                    if ([pwin isVisible] && (ptype == Qt::Window || ptype == Qt::Dialog) && ![qwin parentWindow]) {
-                        NSInteger level = [qwin level];
-                        [pwin addChildWindow:qwin ordered:NSWindowAbove];
-                        if ([qwin level] < level)
-                            [qwin setLevel:level];
-                    }
+                Qt::WindowType ptype = parent->window()->windowType();
+                if ([pwin isVisible] && (ptype == Qt::Window || ptype == Qt::Dialog) && ![qwin parentWindow]) {
+                    NSInteger level = [qwin level];
+                    [pwin addChildWindow:qwin ordered:NSWindowAbove];
+                    if ([qwin level] < level)
+                        [qwin setLevel:level];
                 }
             } else {
                 [pwin removeChildWindow:qwin];
@@ -2845,8 +2847,6 @@ void QWidgetPrivate::setSubWindowStacking(bool set)
         if (child && child->isWindow()) {
             if (NSWindow *cwin = [qt_mac_nativeview_for(child) window]) {
                 if (set) {
-                    if (child->testAttribute(Qt::WA_MacNoCocoaChildWindow))
-                        continue;
                     Qt::WindowType ctype = child->window()->windowType();
                     if ([cwin isVisible] && (ctype == Qt::Window || ctype == Qt::Dialog) && ![cwin parentWindow]) {
                         NSInteger level = [cwin level];