Cocoa: the menu bar is sometimes faulty disabled

In Qt, we allow the menu bar to be enabled if we only have one
exec-ing dialog on screen. That way you can reach the quit menu item to
close down the application. But in order for this to work, some overhaul
of qmenu_mac.mm was need to teach it these rules.

1 files changed, 106 insertions, 49 deletions

diff --git a/src/gui/widgets/qmenu_mac.mm b/src/gui/widgets/qmenu_mac.mm
index 62a482a..3ab6645 100644
--- a/src/gui/widgets/qmenu_mac.mm
+++ b/src/gui/widgets/qmenu_mac.mm
@@ -1906,57 +1906,53 @@ static bool qt_mac_is_ancestor(QWidget* possibleAncestor, QWidget *child)
     Returns true if the entries of menuBar should be disabled,
     based on the modality type of modalWidget.
 */
-static bool qt_mac_should_disable_menu(QMenuBar *menuBar, QWidget *modalWidget)
+static bool qt_mac_should_disable_menu(QMenuBar *menuBar)
 {
-    if (modalWidget == 0 || menuBar == 0)
+    QWidget *modalWidget = qApp->activeModalWidget();
+    if (!modalWidget)
         return false;
 
-    // If there is an application modal window on
-    // screen, the entries of the menubar should be disabled:
+    if (menuBar && menuBar == menubars()->value(modalWidget))
+        // The menu bar is owned by the modal widget.
+        // In that case we should enable it:
+        return false;
+
+    // When there is an application modal window on screen, the entries of
+    // the menubar should be disabled. The exception in Qt is that if the
+    // modal window is the only window on screen, then we enable the menu bar.
     QWidget *w = modalWidget;
+    QWidgetList topLevelWidgets = QApplication::topLevelWidgets();
     while (w) {
-        if (w->isVisible() && w->windowModality() == Qt::ApplicationModal)
-            return true;
+        if (w->isVisible() && w->windowModality() == Qt::ApplicationModal) {
+            for (int i=0; i<topLevelWidgets.size(); ++i) {
+                QWidget *top = topLevelWidgets.at(i);
+                if (w != top && [qt_mac_window_for(top) isVisible]) {
+                    // INVARIANT: we found another visible window
+                    // on screen other than our modalWidget. We therefore
+                    // disable the menu bar to follow normal modality logic:
+                    return true;
+                }
+            }
+            // INVARIANT: We have only one window on screen that happends
+            // to be application modal. We choose to enable the menu bar
+            // in that case to e.g. enable the quit menu item.
+            return false;
+        }
         w = w->parentWidget();
     }
 
     // INVARIANT: modalWidget is window modal. Disable menu entries
-    // if the menu bar belongs to an ancestor of modalWidget:
-    return qt_mac_is_ancestor(menuBar->parentWidget(), modalWidget);
+    // if the menu bar belongs to an ancestor of modalWidget. If menuBar
+    // is nil, we understand it as the default menu bar set by the nib:
+    return menuBar ? qt_mac_is_ancestor(menuBar->parentWidget(), modalWidget) : false;
 }
 
-static void cancelAllMenuTracking()
+static QWidget *findWindowThatShouldDisplayMenubar()
 {
-#ifdef QT_MAC_USE_COCOA
-    QMacCocoaAutoReleasePool pool;
-    NSMenu *mainMenu = [NSApp mainMenu];
-    [mainMenu cancelTracking];
-    for (NSMenuItem *item in [mainMenu itemArray]) {
-        if ([item submenu]) {
-            [[item submenu] cancelTracking];
-        }
-    }
-#endif
-}
-
-/*!
-  \internal
-
-  This function will update the current menu bar and set it as the
-  active menu bar in the Menu Manager.
-
-  \warning This function is not portable.
-
-  \sa QMenu::macMenu(), QMenuBar::macMenu()
-*/
-bool QMenuBar::macUpdateMenuBar()
-{
-    cancelAllMenuTracking();
-    QMenuBar *mb = 0;
-    //find a menu bar
     QWidget *w = qApp->activeWindow();
-
     if (!w) {
+        // We have no active window on screen. Try to
+        // find a window from the list of top levels:
         QWidgetList tlws = QApplication::topLevelWidgets();
         for(int i = 0; i < tlws.size(); ++i) {
             QWidget *tlw = tlws.at(i);
@@ -1967,6 +1963,12 @@ bool QMenuBar::macUpdateMenuBar()
             }
         }
     }
+    return w;
+}
+
+static QMenuBar *findMenubarForWindow(QWidget *w)
+{
+    QMenuBar *mb = 0;
     if (w) {
         mb = menubars()->value(w);
 #ifndef QT_NO_MAINWINDOW
@@ -1980,11 +1982,49 @@ bool QMenuBar::macUpdateMenuBar()
         while(w && !mb)
             mb = menubars()->value((w = w->parentWidget()));
     }
-    if (!mb)
+
+    if (!mb) {
+        // We could not find a menu bar for the window. Lets
+        // check if we have a global (parentless) menu bar instead:
         mb = fallback;
-    //now set it
+    }
+
+    return mb;
+}
+
+static void cancelAllMenuTracking()
+{
+#ifdef QT_MAC_USE_COCOA
+    QMacCocoaAutoReleasePool pool;
+    NSMenu *mainMenu = [NSApp mainMenu];
+    [mainMenu cancelTracking];
+    for (NSMenuItem *item in [mainMenu itemArray]) {
+        if ([item submenu]) {
+            [[item submenu] cancelTracking];
+        }
+    }
+#endif
+}
+
+/*!
+  \internal
+
+  This function will update the current menu bar and set it as the
+  active menu bar in the Menu Manager.
+
+  \warning This function is not portable.
+
+  \sa QMenu::macMenu(), QMenuBar::macMenu()
+*/
+bool QMenuBar::macUpdateMenuBar()
+{
     bool ret = false;
+    cancelAllMenuTracking();
+    QWidget *w = findWindowThatShouldDisplayMenubar();
+    QMenuBar *mb = findMenubarForWindow(w);
+
     if (mb && mb->isNativeMenuBar()) {
+        bool modal = QApplicationPrivate::modalState();
 #ifdef QT_MAC_USE_COCOA
         QMacCocoaAutoReleasePool pool;
 #endif
@@ -2014,16 +2054,18 @@ bool QMenuBar::macUpdateMenuBar()
                 }
             }
 #endif
-            QWidget *modalWidget = qApp->activeModalWidget();
-            if (mb != menubars()->value(modalWidget)) {
-                qt_mac_set_modal_state(menu, qt_mac_should_disable_menu(mb, modalWidget));
-            }
+            // Check if menu is modally shaddowed and should  be disabled:
+            modal = qt_mac_should_disable_menu(mb);
+            if (mb != qt_mac_current_menubar.qmenubar || modal != qt_mac_current_menubar.modal)
+                qt_mac_set_modal_state(menu, modal);
         }
         qt_mac_current_menubar.qmenubar = mb;
-        qt_mac_current_menubar.modal = QApplicationPrivate::modalState();
+        qt_mac_current_menubar.modal = modal;
         ret = true;
     } else if (qt_mac_current_menubar.qmenubar && qt_mac_current_menubar.qmenubar->isNativeMenuBar()) {
-        const bool modal = QApplicationPrivate::modalState();
+        // INVARIANT: The currently active menu bar (if any) is not native. But we do have a
+        // native menu bar from before. So we need to decide whether or not is should be enabled:
+        const bool modal = qt_mac_should_disable_menu(qt_mac_current_menubar.qmenubar);
         if (modal != qt_mac_current_menubar.modal) {
             ret = true;
             if (OSMenuRef menu = qt_mac_current_menubar.qmenubar->macMenu()) {
@@ -2035,14 +2077,29 @@ bool QMenuBar::macUpdateMenuBar()
                 [NSApp setMainMenu:menu];
                 syncMenuBarItemsVisiblity(qt_mac_current_menubar.qmenubar->d_func()->mac_menubar);
 #endif
-                QWidget *modalWidget = qApp->activeModalWidget();
-                if (qt_mac_current_menubar.qmenubar != menubars()->value(modalWidget)) {
-                    qt_mac_set_modal_state(menu, qt_mac_should_disable_menu(mb, modalWidget));
-                }
+                qt_mac_set_modal_state(menu, modal);
             }
             qt_mac_current_menubar.modal = modal;
         }
+    } else {
+        // INVARIANT: There is no menubar specified, so the default one with the application
+        // menu is shown. Check if we need to disable it because of modality. We set
+        // qt_mac_current_menubar.qmenubar to nil to mean the default nib loaded menu:
+#ifdef QT_MAC_USE_COCOA
+        QMacCocoaAutoReleasePool pool;
+        QT_MANGLE_NAMESPACE(QCocoaMenuLoader) *loader = getMenuLoader();
+        OSMenuRef menu = [loader menu];
+        [loader ensureAppMenuInMenu:menu];
+        [NSApp setMainMenu:menu];
+        const bool modal = qt_mac_should_disable_menu(0);
+        if (qt_mac_current_menubar.qmenubar || modal != qt_mac_current_menubar.modal)
+            qt_mac_set_modal_state(menu, modal);
+        qt_mac_current_menubar.qmenubar = 0;
+        qt_mac_current_menubar.modal = modal;
+        ret = true;
+#endif
     }
+
     if(!ret)
         qt_mac_clear_menubar();
     return ret;