QTabWidget documentMode not working correctly on Mac

This feature was implemented in Carbon and was never ported to Cocoa.
The major problem is the fact that Cocoa draws a line under the titlebar,
regardless of what we say. The only way to avoid drawing that line is
by adding a native toolbar and ask it not to draw its baseline. If there
is not toolbar, as it happens in this case, there is no way to prevent
that line from being drawn. So instead we paint over that line and hope
for the best.

Task-number: QTBUG-8159
Reviewed-by: Richard Moe Gustavsen

7 files changed, 227 insertions, 6 deletions

diff --git a/src/gui/kernel/qcocoasharedwindowmethods_mac_p.h b/src/gui/kernel/qcocoasharedwindowmethods_mac_p.h
index e94d247..8652816 100644
--- a/src/gui/kernel/qcocoasharedwindowmethods_mac_p.h
+++ b/src/gui/kernel/qcocoasharedwindowmethods_mac_p.h
@@ -363,3 +363,57 @@ QT_END_NAMESPACE
     }
     [super displayIfNeeded];
 }
+
+// This is a hack and it should be removed once we find the real cause for
+// the painting problems.
+// We have a static variable that signals if we have been called before or not.
+static bool firstDrawingInvocation = true;
+
+// The method below exists only as a workaround to draw/not draw the baseline
+// in the title bar. This is to support unifiedToolbar look.
+
+// This method is very special. To begin with, it is a
+// method that will get called only if we enable documentMode.
+// Furthermore, it won't get called as a normal method, we swap
+// this method with the normal implementation of drawRect in
+// _NSThemeFrame. When this method is active, its mission is to
+// first call the original drawRect implementation so the widget
+// gets proper painting. After that, it needs to detect if there
+// is a toolbar or not, in order to decide how to handle the unified
+// look. The distinction is important since the presence and
+// visibility of a toolbar change the way we enter into unified mode.
+// When there is a toolbar and that toolbar is visible, the problem
+// is as simple as to tell the toolbar not to draw its baseline.
+// However when there is not toolbar or the toolbar is not visible,
+// we need to draw a line on top of the baseline, because the baseline
+// in that case will belong to the title. For this case we need to draw
+// a line on top of the baseline.
+// As usual, there is a special case. When we first are called, we might
+// need to repaint ourselves one more time. We only need that if we
+// didn't get the activation, i.e. when we are launched via the command
+// line. And this only if the toolbar is visible from the beginning,
+// so we have a special flag that signals if we need to repaint or not.
+- (void)drawRectSpecial:(NSRect)rect
+{
+    // Call the original drawing method.
+    [self drawRectOriginal:rect];
+    NSWindow *window = [self window];
+    NSToolbar *toolbar = [window toolbar];
+    if(!toolbar) {
+        // There is no toolbar, we have to draw a line on top of the line drawn by Cocoa.
+        macDrawRectOnTop((void *)window);
+    } else {
+        if([toolbar isVisible]) {
+            // We tell Cocoa to avoid drawing the line at the end.
+            if(firstDrawingInvocation) {
+                firstDrawingInvocation = false;
+                macSyncDrawingOnFirstInvocation((void *)window);
+            } else
+                [toolbar setShowsBaselineSeparator:NO];
+        } else {
+            // There is a toolbar but it is not visible so
+            // we have to draw a line on top of the line drawn by Cocoa.
+            macDrawRectOnTop((void *)window);
+        }
+    }
+}
diff --git a/src/gui/kernel/qt_cocoa_helpers_mac.mm b/src/gui/kernel/qt_cocoa_helpers_mac.mm
index a05c7d5..024c1fc 100644
--- a/src/gui/kernel/qt_cocoa_helpers_mac.mm
+++ b/src/gui/kernel/qt_cocoa_helpers_mac.mm
@@ -1163,15 +1163,81 @@ void qt_mac_updateContentBorderMetricts(void * /*OSWindowRef */window, const ::H
 #endif
 }
 
+#if QT_MAC_USE_COCOA
+void qt_mac_replaceDrawRect(void * /*OSWindowRef */window, QWidgetPrivate *widget)
+{
+    QMacCocoaAutoReleasePool pool;
+    OSWindowRef theWindow = static_cast<OSWindowRef>(window);
+    if(!theWindow)
+        return;
+    id theClass = [[[theWindow contentView] superview] class];
+    // What we do here is basically to add a new selector to NSThemeFrame called
+    // "drawRectOriginal:" which will contain the original implementation of
+    // "drawRect:". After that we get the new implementation from QCocoaWindow
+    // and exchange them. The new implementation is called drawRectSpecial.
+    // We cannot just add the method because it might have been added before and since
+    // we cannot remove a method once it has been added we need to ask QCocoaWindow if
+    // we did the swap or not.
+    if(!widget->drawRectOriginalAdded) {
+        Method m2 = class_getInstanceMethod(theClass, @selector(drawRect:));
+        if(!m2) {
+            // This case is pretty extreme, no drawRect means no drawing!
+            return;
+        }
+        class_addMethod(theClass, @selector(drawRectOriginal:), method_getImplementation(m2), method_getTypeEncoding(m2));
+        widget->drawRectOriginalAdded = true;
+    }
+    if(widget->originalDrawMethod) {
+        Method m0 = class_getInstanceMethod([theWindow class], @selector(drawRectSpecial:));
+        if(!m0) {
+            // Ok, this means the methods were never swapped. Just ignore
+            return;
+        }
+        Method m1 = class_getInstanceMethod(theClass, @selector(drawRect:));
+        if(!m1) {
+            // Ok, this means the methods were never swapped. Just ignore
+            return;
+        }
+        // We have the original method here. Proceed and swap the methods.
+        method_exchangeImplementations(m1, m0);
+        widget->originalDrawMethod = false;
+        [window display];
+    }
+}
+
+void qt_mac_replaceDrawRectOriginal(void * /*OSWindowRef */window, QWidgetPrivate *widget)
+{
+    QMacCocoaAutoReleasePool pool;
+    OSWindowRef theWindow = static_cast<OSWindowRef>(window);
+    id theClass = [[[theWindow contentView] superview] class];
+    // Now we need to revert the methods to their original state.
+    // We cannot remove the method, so we just keep track of it in QCocoaWindow.
+    Method m0 = class_getInstanceMethod([theWindow class], @selector(drawRectSpecial:));
+    if(!m0) {
+        // Ok, this means the methods were never swapped. Just ignore
+        return;
+    }
+    Method m1 = class_getInstanceMethod(theClass, @selector(drawRect:));
+    if(!m1) {
+        // Ok, this means the methods were never swapped. Just ignore
+        return;
+    }
+    method_exchangeImplementations(m1, m0);
+    widget->originalDrawMethod = true;
+    [window display];
+}
+#endif // QT_MAC_USE_COCOA
+
 void qt_mac_showBaseLineSeparator(void * /*OSWindowRef */window, bool show)
 {
+    if(!window)
+        return;
 #if QT_MAC_USE_COCOA
     QMacCocoaAutoReleasePool pool;
     OSWindowRef theWindow = static_cast<OSWindowRef>(window);
     NSToolbar *macToolbar = [theWindow toolbar];
-    if (macToolbar)
-        [macToolbar setShowsBaselineSeparator: show];
-#endif
+    [macToolbar setShowsBaselineSeparator:show];
+#endif // QT_MAC_USE_COCOA
 }
 
 QStringList qt_mac_NSArrayToQStringList(void *nsarray)
@@ -1403,4 +1469,52 @@ void qt_mac_post_retranslateAppMenu()
 #endif
 }
 
+#ifdef QT_MAC_USE_COCOA
+// This method implements the magic for the drawRectSpecial method.
+// We draw a line at the upper edge of the content view in order to
+// override the title baseline.
+void macDrawRectOnTop(void * /*OSWindowRef */window)
+{
+    OSWindowRef theWindow = static_cast<OSWindowRef>(window);
+    NSView *contentView = [theWindow contentView];
+    if(!contentView)
+        return;
+    // Get coordinates of the content view
+    NSRect contentRect = [contentView frame];
+    // Draw a line on top of the already drawn line.
+    // We need to check if we are active or not to use the proper color.
+    if([window isKeyWindow] || [window isMainWindow]) {
+        [[NSColor colorWithCalibratedRed:1.0 green:1.0 blue:1.0 alpha:1.0] set];
+    } else {
+        [[NSColor colorWithCalibratedRed:1.0 green:1.0 blue:1.0 alpha:1.0] set];
+    }
+    NSPoint origin = NSMakePoint(0, contentRect.size.height);
+    NSPoint end = NSMakePoint(contentRect.size.width, contentRect.size.height);
+    [NSBezierPath strokeLineFromPoint:origin toPoint:end];
+}
+
+// This method will (or at least should) get called only once.
+// Its mission is to find out if we are active or not. If we are active
+// we assume that we were launched via finder, otherwise we assume
+// we were called from the command line. The distinction is important,
+// since in the first case we don't need to trigger a paintEvent, while
+// in the second case we do.
+void macSyncDrawingOnFirstInvocation(void * /*OSWindowRef */window)
+{
+    OSWindowRef theWindow = static_cast<OSWindowRef>(window);
+    NSApplication *application = [NSApplication sharedApplication];
+    NSToolbar *toolbar = [window toolbar];
+    if([application isActive]) {
+        // Launched from finder
+        [toolbar setShowsBaselineSeparator:NO];
+    } else {
+        // Launched from commandline
+        [toolbar setVisible:false];
+        [toolbar setShowsBaselineSeparator:NO];
+        [toolbar setVisible:true];
+        [theWindow display];
+    }
+}
+#endif // QT_MAC_USE_COCOA
+
 QT_END_NAMESPACE
diff --git a/src/gui/kernel/qt_cocoa_helpers_mac_p.h b/src/gui/kernel/qt_cocoa_helpers_mac_p.h
index 3fd62a4..5db121a 100644
--- a/src/gui/kernel/qt_cocoa_helpers_mac_p.h
+++ b/src/gui/kernel/qt_cocoa_helpers_mac_p.h
@@ -131,6 +131,8 @@ void macWindowSetHasShadow( void * /*OSWindowRef*/ window, bool hasShadow );
 void macWindowFlush(void * /*OSWindowRef*/ window);
 void macSendToolbarChangeEvent(QWidget *widget);
 void qt_mac_updateContentBorderMetricts(void * /*OSWindowRef */window, const ::HIContentBorderMetrics &metrics);
+void qt_mac_replaceDrawRect(void * /*OSWindowRef */window, QWidgetPrivate *widget);
+void qt_mac_replaceDrawRectOriginal(void * /*OSWindowRef */window, QWidgetPrivate *widget);
 void qt_mac_showBaseLineSeparator(void * /*OSWindowRef */window, bool show);
 void * /*NSImage */qt_mac_create_nsimage(const QPixmap &pm);
 void qt_mac_update_mouseTracking(QWidget *widget);
@@ -140,6 +142,9 @@ void qt_dispatchTabletProximityEvent(void * /*NSEvent * */ tabletEvent);
 #ifdef QT_MAC_USE_COCOA
 bool qt_dispatchKeyEventWithCocoa(void * /*NSEvent * */ keyEvent, QWidget *widgetToGetEvent);
 void qt_cocoaChangeOverrideCursor(const QCursor &cursor);
+// These methods exists only for supporting unified mode.
+void macDrawRectOnTop(void * /*OSWindowRef */ window);
+void macSyncDrawingOnFirstInvocation(void * /*OSWindowRef */window);
 #endif
 void qt_mac_menu_collapseSeparators(void * /*NSMenu */ menu, bool collapse);
 bool qt_dispatchKeyEvent(void * /*NSEvent * */ keyEvent, QWidget *widgetToGetEvent);
diff --git a/src/gui/kernel/qt_mac_p.h b/src/gui/kernel/qt_mac_p.h
index 7bfb257..3341ce1 100644
--- a/src/gui/kernel/qt_mac_p.h
+++ b/src/gui/kernel/qt_mac_p.h
@@ -57,6 +57,7 @@
 
 #ifdef __OBJC__
 #include <Cocoa/Cocoa.h>
+#include <objc/runtime.h>
 #endif
 
 #include <CoreServices/CoreServices.h>
diff --git a/src/gui/kernel/qwidget.cpp b/src/gui/kernel/qwidget.cpp
index 441e823..cf952aa 100644
--- a/src/gui/kernel/qwidget.cpp
+++ b/src/gui/kernel/qwidget.cpp
@@ -220,6 +220,11 @@ QWidgetPrivate::QWidgetPrivate(int version)
 
     isWidget = true;
     memset(high_attributes, 0, sizeof(high_attributes));
+#if QT_MAC_USE_COCOA
+    drawRectOriginalAdded = false;
+    originalDrawMethod = true;
+    changeMethods = false;
+#endif // QT_MAC_USE_COCOA
 #ifdef QWIDGET_EXTRA_DEBUG
     static int count = 0;
     qDebug() << "widgets" << ++count;
@@ -12292,6 +12297,28 @@ void QWidgetPrivate::_q_delayedDestroy(WId winId)
 }
 #endif
 
+#if QT_MAC_USE_COCOA
+void QWidgetPrivate::syncUnifiedMode() {
+    // The whole purpose of this method is to keep the unifiedToolbar in sync.
+    // That means making sure we either exchange the drawing methods or we let
+    // the toolbar know that it does not require to draw the baseline.
+    Q_Q(QWidget);
+    // This function makes sense only if this is a top level
+    if(!q->isWindow())
+        return;
+    OSWindowRef window = qt_mac_window_for(q);
+    if(changeMethods) {
+        // Ok, we are in documentMode.
+        if(originalDrawMethod)
+            qt_mac_replaceDrawRect(window, this);
+    } else {
+        if(!originalDrawMethod)
+            qt_mac_replaceDrawRectOriginal(window, this);
+    }
+}
+
+#endif // QT_MAC_USE_COCOA
+
 QT_END_NAMESPACE
 
 #include "moc_qwidget.cpp"
diff --git a/src/gui/kernel/qwidget_p.h b/src/gui/kernel/qwidget_p.h
index 89ea256..cad60b5 100644
--- a/src/gui/kernel/qwidget_p.h
+++ b/src/gui/kernel/qwidget_p.h
@@ -773,6 +773,13 @@ public:
     void finishCreateWindow_sys_Cocoa(void * /*NSWindow * */ windowRef);
     void syncCocoaMask();
     void finishCocoaMaskSetup();
+    void syncUnifiedMode();
+    // Did we add the drawRectOriginal method?
+    bool drawRectOriginalAdded;
+    // Is the original drawRect method available?
+    bool originalDrawMethod;
+    // Do we need to change the methods?
+    bool changeMethods;
 #endif
     void determineWindowClass();
     void transferChildren();
diff --git a/src/gui/widgets/qtabbar.cpp b/src/gui/widgets/qtabbar.cpp
index d03a2f4..8aaaade 100644
--- a/src/gui/widgets/qtabbar.cpp
+++ b/src/gui/widgets/qtabbar.cpp
@@ -69,6 +69,7 @@
 
 QT_BEGIN_NAMESPACE
 
+
 inline static bool verticalTabs(QTabBar::Shape shape)
 {
     return shape == QTabBar::RoundedWest
@@ -95,9 +96,20 @@ void QTabBarPrivate::updateMacBorderMetrics()
         metrics.left = 0;
         metrics.right = 0;
         qt_mac_updateContentBorderMetricts(window, metrics);
-        
-        // hide the base line separator if the tabs have docuemnt mode enabled (Cocoa)
-        qt_mac_showBaseLineSeparator(window, !documentMode);
+#if QT_MAC_USE_COCOA
+        // In Cocoa we need to keep track of the drawRect method.
+        // If documentMode is enabled we need to change it, unless
+        // a toolbar is present.
+        // Notice that all the information is kept in the window,
+        // that's why we get the private widget for it instead of
+        // the private widget for this widget.
+        QWidgetPrivate *privateWidget = qt_widget_private(q->window());
+        if(privateWidget)
+            privateWidget->changeMethods = documentMode;
+        // Since in Cocoa there is no simple way to remove the baseline, so we just ask the
+        // top level to do the magic for us.
+        privateWidget->syncUnifiedMode();
+#endif // QT_MAC_USE_COCOA
     }
 #endif
 }
@@ -2193,6 +2205,7 @@ bool QTabBar::documentMode() const
 void QTabBar::setDocumentMode(bool enabled)
 {
     Q_D(QTabBar);
+
     d->documentMode = enabled;
     d->updateMacBorderMetrics();
 }