QuickTime media backend: Render CIImage based video frames directly with

OpenGL paint engine.
This makes video rendering with QGraphicsVideoItem much faster on
64 bits systems.

Reviewed-by: Justin McPherson

6 files changed, 249 insertions, 42 deletions

diff --git a/src/plugins/mediaservices/qt7/qt7.pro b/src/plugins/mediaservices/qt7/qt7.pro
index 6624d13..8791d73 100644
--- a/src/plugins/mediaservices/qt7/qt7.pro
+++ b/src/plugins/mediaservices/qt7/qt7.pro
@@ -25,6 +25,7 @@ HEADERS += \
     qt7movieviewrenderer.h \
     qt7serviceplugin.h \
     qt7movierenderer.h \
+    qt7ciimagevideobuffer.h \
     qcvdisplaylink.h
 
 OBJECTIVE_SOURCES += \
@@ -35,6 +36,7 @@ OBJECTIVE_SOURCES += \
     qt7movieviewrenderer.mm \
     qt7movierenderer.mm \
     qt7videooutputcontrol.mm \
+    qt7ciimagevideobuffer.mm \
     qcvdisplaylink.mm
 
 include(mediaplayer/mediaplayer.pri)
diff --git a/src/plugins/mediaservices/qt7/qt7ciimagevideobuffer.h b/src/plugins/mediaservices/qt7/qt7ciimagevideobuffer.h
new file mode 100644
index 0000000..669724f
--- /dev/null
+++ b/src/plugins/mediaservices/qt7/qt7ciimagevideobuffer.h
@@ -0,0 +1,90 @@
+/****************************************************************************
+**
+** Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies).
+** All rights reserved.
+** Contact: Nokia Corporation (qt-info@nokia.com)
+**
+** This file is part of the plugins of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:LGPL$
+** No Commercial Usage
+** This file contains pre-release code and may not be distributed.
+** You may use this file in accordance with the terms and conditions
+** contained in the Technology Preview License Agreement accompanying
+** this package.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 2.1 as published by the Free Software
+** Foundation and appearing in the file LICENSE.LGPL included in the
+** packaging of this file.  Please review the following information to
+** ensure the GNU Lesser General Public License version 2.1 requirements
+** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+**
+** In addition, as a special exception, Nokia gives you certain additional
+** rights.  These rights are described in the Nokia Qt LGPL Exception
+** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
+**
+** If you have questions regarding the use of this file, please contact
+** Nokia at qt-info@nokia.com.
+**
+**
+**
+**
+**
+**
+**
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+#ifndef QT7CIIMAGEVIDEOBUFFER_H
+#define QT7CIIMAGEVIDEOBUFFER_H
+
+#include "qt7backend.h"
+#import <QTKit/QTKit.h>
+
+#include <QtCore/qvariant.h>
+#include <QtMultimedia/qabstractvideobuffer.h>
+
+
+//
+//  W A R N I N G
+//  -------------
+//
+// This file is not part of the Qt API. It exists purely as an
+// implementation detail. This header file may change from version to
+// version without notice, or even be removed.
+//
+// We mean it.
+//
+
+QT_BEGIN_HEADER
+
+QT_BEGIN_NAMESPACE
+
+class QT7CIImageVideoBuffer : public QAbstractVideoBuffer
+{
+public:
+    QT7CIImageVideoBuffer(CIImage *image);
+
+    virtual ~QT7CIImageVideoBuffer();
+
+    MapMode mapMode() const;
+    uchar *map(MapMode mode, int *numBytes, int *bytesPerLine);
+    void unmap();
+    QVariant handle() const;
+
+private:
+    CIImage *m_image;
+    NSBitmapImageRep *m_buffer;
+    MapMode m_mode;
+};
+
+
+QT_END_NAMESPACE
+
+QT_END_HEADER
+
+#endif
diff --git a/src/plugins/mediaservices/qt7/qt7ciimagevideobuffer.mm b/src/plugins/mediaservices/qt7/qt7ciimagevideobuffer.mm
new file mode 100644
index 0000000..3778b58
--- /dev/null
+++ b/src/plugins/mediaservices/qt7/qt7ciimagevideobuffer.mm
@@ -0,0 +1,104 @@
+/****************************************************************************
+**
+** Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies).
+** All rights reserved.
+** Contact: Nokia Corporation (qt-info@nokia.com)
+**
+** This file is part of the plugins of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:LGPL$
+** No Commercial Usage
+** This file contains pre-release code and may not be distributed.
+** You may use this file in accordance with the terms and conditions
+** contained in the Technology Preview License Agreement accompanying
+** this package.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 2.1 as published by the Free Software
+** Foundation and appearing in the file LICENSE.LGPL included in the
+** packaging of this file.  Please review the following information to
+** ensure the GNU Lesser General Public License version 2.1 requirements
+** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+**
+** In addition, as a special exception, Nokia gives you certain additional
+** rights.  These rights are described in the Nokia Qt LGPL Exception
+** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
+**
+** If you have questions regarding the use of this file, please contact
+** Nokia at qt-info@nokia.com.
+**
+**
+**
+**
+**
+**
+**
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+#include "qt7ciimagevideobuffer.h"
+
+QT7CIImageVideoBuffer::QT7CIImageVideoBuffer(CIImage *image)
+    : QAbstractVideoBuffer(CoreImageHandle)
+    , m_image(image)
+    , m_buffer(0)
+    , m_mode(NotMapped)
+{
+    [m_image retain];
+}
+
+QT7CIImageVideoBuffer::~QT7CIImageVideoBuffer()
+{
+    [m_image release];
+    [m_buffer release];
+}
+
+QAbstractVideoBuffer::MapMode QT7CIImageVideoBuffer::mapMode() const
+{
+    return m_mode;
+}
+
+uchar *QT7CIImageVideoBuffer::map(QAbstractVideoBuffer::MapMode mode, int *numBytes, int *bytesPerLine)
+{
+    if (mode == NotMapped || m_mode != NotMapped || !m_image)
+        return 0;
+
+    if (!m_buffer) {
+        //swap R and B channels
+        CIFilter *colorSwapFilter = [CIFilter filterWithName: @"CIColorMatrix"  keysAndValues:
+                                     @"inputImage", m_image,
+                                     @"inputRVector", [CIVector vectorWithX: 0  Y: 0  Z: 1  W: 0],
+                                     @"inputGVector", [CIVector vectorWithX: 0  Y: 1  Z: 0  W: 0],
+                                     @"inputBVector", [CIVector vectorWithX: 1  Y: 0  Z: 0  W: 0],
+                                     @"inputAVector", [CIVector vectorWithX: 0  Y: 0  Z: 0  W: 1],
+                                     @"inputBiasVector", [CIVector vectorWithX: 0  Y: 0  Z: 0  W: 0],
+                                     nil];
+        CIImage *img = [colorSwapFilter valueForKey: @"outputImage"];
+
+        m_buffer = [[NSBitmapImageRep alloc] initWithCIImage:img];
+    }
+
+    if (numBytes)
+        *numBytes = [m_buffer bytesPerPlane];
+
+    if (bytesPerLine)
+        *bytesPerLine = [m_buffer bytesPerRow];
+
+    m_mode = mode;
+
+    return [m_buffer bitmapData];
+}
+
+void QT7CIImageVideoBuffer::unmap()
+{
+    m_mode = NotMapped;
+}
+
+QVariant QT7CIImageVideoBuffer::handle() const
+{
+    return QVariant::fromValue<void*>(m_image);
+}
+
diff --git a/src/plugins/mediaservices/qt7/qt7movierenderer.mm b/src/plugins/mediaservices/qt7/qt7movierenderer.mm
index 1c1f5e4..a6d9d8d 100644
--- a/src/plugins/mediaservices/qt7/qt7movierenderer.mm
+++ b/src/plugins/mediaservices/qt7/qt7movierenderer.mm
@@ -46,6 +46,7 @@
 #include "qt7playercontrol.h"
 #include "qt7movierenderer.h"
 #include "qt7playersession.h"
+#include "qt7ciimagevideobuffer.h"
 #include "qcvdisplaylink.h"
 #include <QtCore/qdebug.h>
 #include <QtCore/qcoreapplication.h>
@@ -64,7 +65,7 @@ class CVGLTextureVideoBuffer : public QAbstractVideoBuffer
 {
 public:
     CVGLTextureVideoBuffer(CVOpenGLTextureRef buffer)
-        : QAbstractVideoBuffer(NoHandle)
+        : QAbstractVideoBuffer(GLTextureHandle)
         , m_buffer(buffer)
         , m_mode(NotMapped)
     {        
@@ -82,11 +83,6 @@ public:
         return QVariant(int(id));
     }
 
-    HandleType handleType() const
-    {
-        return GLTextureHandle;
-    }
-
     MapMode mapMode() const { return m_mode; }
 
     uchar *map(MapMode mode, int *numBytes, int *bytesPerLine)
@@ -447,15 +443,14 @@ void QT7MovieRenderer::updateVideoFrame(const CVTimeStamp &ts)
         OSStatus status = QTVisualContextCopyImageForTime(m_visualContext, NULL, &ts, &imageBuffer);
 
         if (status == noErr && imageBuffer) {
-            //qDebug() << "render video frame";
             QAbstractVideoBuffer *buffer = 0;
 
             if (m_usingGLContext) {
-                buffer = new CVGLTextureVideoBuffer((CVOpenGLTextureRef)imageBuffer);
+                buffer = new QT7CIImageVideoBuffer([CIImage imageWithCVImageBuffer:imageBuffer]);
                 CVOpenGLTextureRelease((CVOpenGLTextureRef)imageBuffer);
-                //qDebug() << "render GL video frame" << buffer->handle();
             } else {
                 buffer = new CVPixelBufferVideoBuffer((CVPixelBufferRef)imageBuffer);
+                //buffer = new QT7CIImageVideoBuffer( [CIImage imageWithCVImageBuffer:imageBuffer] );
                 CVPixelBufferRelease((CVPixelBufferRef)imageBuffer);
             }
 
diff --git a/src/plugins/mediaservices/qt7/qt7movieviewrenderer.h b/src/plugins/mediaservices/qt7/qt7movieviewrenderer.h
index 336006c..0126360 100644
--- a/src/plugins/mediaservices/qt7/qt7movieviewrenderer.h
+++ b/src/plugins/mediaservices/qt7/qt7movieviewrenderer.h
@@ -87,6 +87,7 @@ private:
     QSize m_nativeSize;
     QAbstractVideoSurface *m_surface;
     QVideoFrame m_currentFrame;
+    bool m_pendingRenderEvent;
     QMutex m_mutex;
 };
 
diff --git a/src/plugins/mediaservices/qt7/qt7movieviewrenderer.mm b/src/plugins/mediaservices/qt7/qt7movieviewrenderer.mm
index 5047853..5f11479 100644
--- a/src/plugins/mediaservices/qt7/qt7movieviewrenderer.mm
+++ b/src/plugins/mediaservices/qt7/qt7movieviewrenderer.mm
@@ -46,6 +46,7 @@
 #include "qt7playercontrol.h"
 #include "qt7movieviewrenderer.h"
 #include "qt7playersession.h"
+#include "qt7ciimagevideobuffer.h"
 #include <QtCore/qdebug.h>
 #include <QtCore/qcoreevent.h>
 #include <QtCore/qcoreapplication.h>
@@ -113,6 +114,7 @@ QT_END_NAMESPACE
 - (HiddenQTMovieView *) initWithRenderer:(QT7MovieViewRenderer *)renderer;
 - (void) setRenderer:(QT7MovieViewRenderer *)renderer;
 - (void) setDrawRect:(const QRect &)rect;
+- (CIImage *) view:(QTMovieView *)view willDisplayImage:(CIImage *)img;
 @end
 
 @implementation HiddenQTMovieView
@@ -163,33 +165,37 @@ QT_END_NAMESPACE
     // before the image will be drawn.
     Q_UNUSED(view);
     if (m_renderer) {
-        NSBitmapImageRep *bitmap = [[NSBitmapImageRep alloc] initWithCIImage:img];
         CGRect bounds = [img extent];
         int w = bounds.size.width;
         int h = bounds.size.height;
 
-        // Swap red and blue (same as QImage::rgbSwapped, but without copy)
-        uchar *data = [bitmap bitmapData];
-        //qDebug() << data << w << h;
-        int bytesPerLine = [bitmap bytesPerRow];
-        for (int i=0; i<h; ++i) {
-            quint32 *p = (quint32*)data;
-            data += bytesPerLine;
-            quint32 *end = p + w;
-            while (p < end) {
-                *p = ((*p << 16) & 0xff0000) | ((*p >> 16) & 0xff) | (*p & 0xff00ff00);
-                p++;
-            }
-        }
+        QVideoFrame frame;
 
-        QVideoFrame frame( new NSBitmapVideoBuffer(bitmap), QSize(w,h), QVideoFrame::Format_RGB32 );
+        QAbstractVideoSurface *surface = m_renderer->surface();
+        if (!surface || !surface->isActive())
+            return img;
 
-        //static int i=0;
-        //i++;
-        //QImage img([bitmap bitmapData], w, h, QImage::Format_RGB32);
-        //img.save(QString("img%1.jpg").arg(i));
-
-        [bitmap release];
+        if (surface->surfaceFormat().handleType() == QAbstractVideoBuffer::CoreImageHandle) {
+            //surface supports rendering of opengl based CIImage
+            frame = QVideoFrame(new QT7CIImageVideoBuffer(img), QSize(w,h), QVideoFrame::Format_RGB32 );
+        } else {
+            //Swap R and B colors
+            CIFilter *colorSwapFilter = [CIFilter filterWithName: @"CIColorMatrix"  keysAndValues:
+                                         @"inputImage", img,
+                                         @"inputRVector", [CIVector vectorWithX: 0  Y: 0  Z: 1  W: 0],
+                                         @"inputGVector", [CIVector vectorWithX: 0  Y: 1  Z: 0  W: 0],
+                                         @"inputBVector", [CIVector vectorWithX: 1  Y: 0  Z: 0  W: 0],
+                                         @"inputAVector", [CIVector vectorWithX: 0  Y: 0  Z: 0  W: 1],
+                                         @"inputBiasVector", [CIVector vectorWithX: 0  Y: 0  Z: 0  W: 0],
+                                         nil];
+            CIImage *img = [colorSwapFilter valueForKey: @"outputImage"];
+            NSBitmapImageRep *bitmap =[[NSBitmapImageRep alloc] initWithCIImage:img];
+            //requesting the bitmap data is slow,
+            //but it's better to do it here to avoid blocking the main thread for a long.
+            [bitmap bitmapData];
+            frame = QVideoFrame(new NSBitmapVideoBuffer(bitmap), QSize(w,h), QVideoFrame::Format_RGB32 );
+            [bitmap release];
+        }
 
         if (m_renderer)
             m_renderer->renderFrame(frame);
@@ -230,7 +236,8 @@ QT7MovieViewRenderer::QT7MovieViewRenderer(QObject *parent)
    :QT7VideoRendererControl(parent),
     m_movie(0),
     m_movieView(0),
-    m_surface(0)
+    m_surface(0),
+    m_pendingRenderEvent(false)
 {    
 }
 
@@ -267,11 +274,15 @@ void QT7MovieViewRenderer::setupVideoOutput()
         }
 
         [movieView setMovie:(QTMovie*)m_movie];
-        //[movieView setDrawRect:QRect(QPoint(0,0), m_nativeSize)];
+        [movieView setDrawRect:QRect(QPoint(0,0), m_nativeSize)];
     }
 
     if (m_surface && !m_nativeSize.isEmpty()) {
-        QVideoSurfaceFormat format(m_nativeSize, QVideoFrame::Format_RGB32);
+        bool coreImageFrameSupported = !m_surface->supportedPixelFormats(QAbstractVideoBuffer::CoreImageHandle).isEmpty() &&
+                                       !m_surface->supportedPixelFormats(QAbstractVideoBuffer::GLTextureHandle).isEmpty();
+
+        QVideoSurfaceFormat format(m_nativeSize, QVideoFrame::Format_RGB32,
+                                   coreImageFrameSupported ? QAbstractVideoBuffer::CoreImageHandle : QAbstractVideoBuffer::NoHandle);
 
         if (m_surface->isActive() && m_surface->surfaceFormat() != format) {
 //            qDebug() << "Surface format was changed, stop the surface.";
@@ -279,10 +290,11 @@ void QT7MovieViewRenderer::setupVideoOutput()
         }
 
         if (!m_surface->isActive()) {
-//            qDebug() << "Starting the surface with format" << format;
-            m_surface->start(format);
-//            if (!m_surface->start(format))
-//                qDebug() << "failed to start video surface" << m_surface->error();
+            //qDebug() << "Starting the surface with format" << format;
+            if (!m_surface->start(format)) {
+                qWarning() << "Failed to start video surface" << m_surface->error();
+                qWarning() << "Surface format:" << format;
+            }
         }
     }
 }
@@ -330,18 +342,21 @@ void QT7MovieViewRenderer::setSurface(QAbstractVideoSurface *surface)
 
 void QT7MovieViewRenderer::renderFrame(const QVideoFrame &frame)
 {
-    {
-        QMutexLocker locker(&m_mutex);
-        m_currentFrame = frame;
-    }
 
-    qApp->postEvent(this, new QEvent(QEvent::User), Qt::HighEventPriority);
+    QMutexLocker locker(&m_mutex);
+    m_currentFrame = frame;
+
+    if (!m_pendingRenderEvent)
+        qApp->postEvent(this, new QEvent(QEvent::User), Qt::HighEventPriority);
+
+    m_pendingRenderEvent = true;
 }
 
 bool QT7MovieViewRenderer::event(QEvent *event)
 {
     if (event->type() == QEvent::User) {
         QMutexLocker locker(&m_mutex);
+        m_pendingRenderEvent = false;
         if (m_surface->isActive())
             m_surface->present(m_currentFrame);
     }