QNAM Zerocopy: QNAM implementation part

Reviewed-by: Peter Hartmann
Reviewed-by: Jeremy Katz
Reviewed-by: Aleksandar Sasha Babic

6 files changed, 138 insertions, 1 deletions

diff --git a/src/network/access/qnetworkaccessbackend.cpp b/src/network/access/qnetworkaccessbackend.cpp
index 60f7dc6..45495f7 100644
--- a/src/network/access/qnetworkaccessbackend.cpp
+++ b/src/network/access/qnetworkaccessbackend.cpp
@@ -252,6 +252,17 @@ void QNetworkAccessBackend::writeDownstreamData(QIODevice *data)
     reply->appendDownstreamData(data);
 }
 
+// not actually appending data, it was already written to the user buffer
+void QNetworkAccessBackend::writeDownstreamDataDownloadBuffer(qint64 bytesReceived, qint64 bytesTotal)
+{
+    reply->appendDownstreamDataDownloadBuffer(bytesReceived, bytesTotal);
+}
+
+char* QNetworkAccessBackend::getDownloadBuffer(qint64 size)
+{
+    return reply->getDownloadBuffer(size);
+}
+
 QVariant QNetworkAccessBackend::header(QNetworkRequest::KnownHeaders header) const
 {
     return reply->q_func()->header(header);
diff --git a/src/network/access/qnetworkaccessbackend_p.h b/src/network/access/qnetworkaccessbackend_p.h
index 4fe6de6..9f8a01f 100644
--- a/src/network/access/qnetworkaccessbackend_p.h
+++ b/src/network/access/qnetworkaccessbackend_p.h
@@ -177,6 +177,10 @@ protected:
     qint64 nextDownstreamBlockSize() const;
     void writeDownstreamData(QByteDataBuffer &list);
 
+    // not actually appending data, it was already written to the user buffer
+    void writeDownstreamDataDownloadBuffer(qint64, qint64);
+    char* getDownloadBuffer(qint64);
+
 public slots:
     // for task 251801, needs to be a slot to be called asynchronously
     void writeDownstreamData(QIODevice *data);
diff --git a/src/network/access/qnetworkaccesshttpbackend.cpp b/src/network/access/qnetworkaccesshttpbackend.cpp
index f617244..8b9a99f 100644
--- a/src/network/access/qnetworkaccesshttpbackend.cpp
+++ b/src/network/access/qnetworkaccesshttpbackend.cpp
@@ -542,6 +542,9 @@ void QNetworkAccessHttpBackend::postRequest()
         break;                  // can't happen
     }
 
+    bool encrypt = (url().scheme().toLower() == QLatin1String("https"));
+    httpRequest.setSsl(encrypt);
+
     httpRequest.setUrl(url());
 
     QList<QByteArray> headers = request().rawHeaderList();
@@ -595,7 +598,6 @@ void QNetworkAccessHttpBackend::postRequest()
     httpReply->ignoreSslErrors(pendingIgnoreSslErrorsList);
 #endif
 
-    connect(httpReply, SIGNAL(readyRead()), SLOT(replyReadyRead()));
     connect(httpReply, SIGNAL(finished()), SLOT(replyFinished()));
     connect(httpReply, SIGNAL(finishedWithError(QNetworkReply::NetworkError,QString)),
             SLOT(httpError(QNetworkReply::NetworkError,QString)));
@@ -859,9 +861,33 @@ void QNetworkAccessHttpBackend::replyHeaderChanged()
         if (!isCachingEnabled())
             setCachingEnabled(true);
     }
+
+    // Check if a download buffer is supported from the HTTP reply
+    char *buf = 0;
+    if (httpReply->supportsUserProvidedDownloadBuffer()) {
+        // Check if a download buffer is supported by the user
+        buf = getDownloadBuffer(httpReply->contentLength());
+        if (buf) {
+            httpReply->setUserProvidedDownloadBuffer(buf);
+            // If there is a download buffer we react on the progress signal
+            connect(httpReply, SIGNAL(dataReadProgress(int,int)), SLOT(replyDownloadProgressSlot(int,int)));
+        }
+    }
+
+    // If there is no buffer, we react on the readyRead signal
+    if (!buf) {
+        connect(httpReply, SIGNAL(readyRead()), SLOT(replyReadyRead()));
+    }
+
     metaDataChanged();
 }
 
+void QNetworkAccessHttpBackend::replyDownloadProgressSlot(int received,  int total)
+{
+    // we can be sure here that there is a download buffer
+    writeDownstreamDataDownloadBuffer(received, total);
+}
+
 void QNetworkAccessHttpBackend::httpAuthenticationRequired(const QHttpNetworkRequest &,
                                                            QAuthenticator *auth)
 {
@@ -1169,6 +1195,11 @@ bool QNetworkAccessHttpBackend::canResume() const
             return false;
     }
 
+    // If we're using a download buffer then we don't support resuming/migration
+    // right now. Too much trouble.
+    if (httpReply->userProvidedDownloadBuffer())
+        return false;
+
     return true;
 }
 
diff --git a/src/network/access/qnetworkaccesshttpbackend_p.h b/src/network/access/qnetworkaccesshttpbackend_p.h
index c4c88ae..fb12781 100644
--- a/src/network/access/qnetworkaccesshttpbackend_p.h
+++ b/src/network/access/qnetworkaccesshttpbackend_p.h
@@ -104,6 +104,7 @@ private slots:
     void replyReadyRead();
     void replyFinished();
     void replyHeaderChanged();
+    void replyDownloadProgressSlot(int,int);
     void httpAuthenticationRequired(const QHttpNetworkRequest &request, QAuthenticator *auth);
     void httpCacheCredentials(const QHttpNetworkRequest &request, QAuthenticator *auth);
     void httpError(QNetworkReply::NetworkError error, const QString &errorString);
diff --git a/src/network/access/qnetworkreplyimpl.cpp b/src/network/access/qnetworkreplyimpl.cpp
index 3798ac2..c8808af 100644
--- a/src/network/access/qnetworkreplyimpl.cpp
+++ b/src/network/access/qnetworkreplyimpl.cpp
@@ -49,11 +49,15 @@
 #include "QtNetwork/qnetworksession.h"
 #include "qnetworkaccesshttpbackend_p.h"
 #include "qnetworkaccessmanager_p.h"
+#include <QVarLengthArray>
 
 #include <QtCore/QCoreApplication>
 
 QT_BEGIN_NAMESPACE
 
+typedef QSharedPointer<QVarLengthArray<char, 0> > QVarLengthArraySharedPointer;
+Q_DECLARE_METATYPE(QVarLengthArraySharedPointer)
+
 inline QNetworkReplyImplPrivate::QNetworkReplyImplPrivate()
     : backend(0), outgoingData(0), outgoingDataBuffer(0),
       copyDevice(0),
@@ -62,6 +66,8 @@ inline QNetworkReplyImplPrivate::QNetworkReplyImplPrivate()
       bytesDownloaded(0), lastBytesDownloaded(-1), bytesUploaded(-1), preMigrationDownloaded(-1),
       httpStatusCode(0),
       state(Idle)
+      , downloadBuffer(0)
+      , downloadBufferPosition(0)
 {
 }
 
@@ -125,6 +131,10 @@ void QNetworkReplyImplPrivate::_q_copyReadyRead()
     if (!copyDevice || !q->isOpen())
         return;
 
+    // FIXME Optimize to use download buffer if it is a QBuffer.
+    // Needs to be done where sendCacheContents() (?) of HTTP is emitting
+    // metaDataChanged ?
+
     forever {
         qint64 bytesToRead = nextDownstreamBlockSize();
         if (bytesToRead == 0)
@@ -590,6 +600,55 @@ void QNetworkReplyImplPrivate::appendDownstreamData(const QByteArray &data)
     qFatal("QNetworkReplyImplPrivate::appendDownstreamData not implemented");
 }
 
+char* QNetworkReplyImplPrivate::getDownloadBuffer(qint64 size)
+{
+    Q_Q(QNetworkReplyImpl);
+
+    // Check attribute() if allocating a buffer of that size can be allowed
+    if (!downloadBuffer) {
+        QVariant bufferAllocationPolicy = request.attribute(QNetworkRequest::MaximumDownloadBufferSizeAttribute);
+        if (bufferAllocationPolicy.isValid() && bufferAllocationPolicy.toLongLong() >= size) {
+            downloadBufferArray = QSharedPointer<QVarLengthArray<char, 0> >(new QVarLengthArray<char, 0>());
+            downloadBufferArray->reserve(size);
+
+            downloadBuffer = downloadBufferArray->data();
+
+            q->setAttribute(QNetworkRequest::DownloadBufferAttribute, qVariantFromValue<QSharedPointer<QVarLengthArray<char, 0> > > (downloadBufferArray));
+        }
+    }
+
+    return downloadBuffer;
+}
+
+void QNetworkReplyImplPrivate::appendDownstreamDataDownloadBuffer(qint64 bytesReceived, qint64 bytesTotal)
+{
+    Q_Q(QNetworkReplyImpl);
+    if (!q->isOpen())
+        return;
+
+    if (cacheEnabled && !cacheSaveDevice)
+        initCacheSaveDevice();
+
+    if (cacheSaveDevice && bytesReceived == bytesTotal) {
+//        if (lastBytesDownloaded == -1)
+//            lastBytesDownloaded = 0;
+//        cacheSaveDevice->write(downloadBuffer + lastBytesDownloaded, bytesReceived - lastBytesDownloaded);
+
+        // Write everything in one go if we use a download buffer. might be more performant.
+        cacheSaveDevice->write(downloadBuffer, bytesTotal);
+    }
+
+    bytesDownloaded = bytesReceived;
+    lastBytesDownloaded = bytesReceived;
+
+    // Update the array so our user (e.g. QtWebKit) knows the real size
+    if (bytesReceived > 0)
+        downloadBufferArray->resize(bytesReceived);
+
+    emit q->downloadProgress(bytesDownloaded, bytesTotal);
+    emit q->readyRead();
+}
+
 void QNetworkReplyImplPrivate::finished()
 {
     Q_Q(QNetworkReplyImpl);
@@ -784,6 +843,13 @@ bool QNetworkReplyImpl::canReadLine () const
 */
 qint64 QNetworkReplyImpl::bytesAvailable() const
 {
+    // Special case for the "zero copy" download buffer
+    Q_D(const QNetworkReplyImpl);
+    if (d->downloadBuffer) {
+        qint64 maxAvail = d->downloadBufferArray->size() - d->downloadBufferPosition;
+        return QNetworkReply::bytesAvailable() + maxAvail;
+    }
+
     return QNetworkReply::bytesAvailable() + d_func()->readBuffer.byteAmount();
 }
 
@@ -838,8 +904,22 @@ void QNetworkReplyImpl::ignoreSslErrorsImplementation(const QList<QSslError> &er
 qint64 QNetworkReplyImpl::readData(char *data, qint64 maxlen)
 {
     Q_D(QNetworkReplyImpl);
+
+    // Special case code if we have the "zero copy" download buffer
+    if (d->downloadBuffer) {
+        qint64 maxAvail = qMin<qint64>(d->downloadBufferArray->size() - d->downloadBufferPosition, maxlen);
+        if (maxAvail == 0)
+            return d->state == QNetworkReplyImplPrivate::Finished ? -1 : 0;
+        // FIXME what about "Aborted" state?
+        qMemCopy(data, d->downloadBuffer + d->downloadBufferPosition, maxAvail);
+        d->downloadBufferPosition += maxAvail;
+        return maxAvail;
+    }
+
+
     if (d->readBuffer.isEmpty())
         return d->state == QNetworkReplyImplPrivate::Finished ? -1 : 0;
+    // FIXME what about "Aborted" state?
 
     d->backendNotify(QNetworkReplyImplPrivate::NotifyDownstreamReadyWrite);
     if (maxlen == 1) {
diff --git a/src/network/access/qnetworkreplyimpl_p.h b/src/network/access/qnetworkreplyimpl_p.h
index 38084bd..ab11ebe 100644
--- a/src/network/access/qnetworkreplyimpl_p.h
+++ b/src/network/access/qnetworkreplyimpl_p.h
@@ -62,6 +62,7 @@
 #include "QtCore/qbuffer.h"
 #include "private/qringbuffer_p.h"
 #include "private/qbytedata_p.h"
+#include <QVarLengthArray>
 
 QT_BEGIN_NAMESPACE
 
@@ -163,6 +164,9 @@ public:
     void appendDownstreamData(QIODevice *data);
     void appendDownstreamData(const QByteArray &data);
 
+    char* getDownloadBuffer(qint64 size);
+    void appendDownstreamDataDownloadBuffer(qint64, qint64);
+
     void finished();
     void error(QNetworkReply::NetworkError code, const QString &errorString);
     void metaDataChanged();
@@ -191,6 +195,7 @@ public:
     QList<QNetworkProxy> proxyList;
 #endif
 
+    // Used for normal downloading. For "zero copy" the downloadBuffer is used
     QByteDataBuffer readBuffer;
     qint64 bytesDownloaded;
     qint64 lastBytesDownloaded;
@@ -202,6 +207,11 @@ public:
 
     State state;
 
+    // only used when the "zero copy" style is used. Else readBuffer is used.
+    QSharedPointer< QVarLengthArray<char, 0> > downloadBufferArray;
+    char* downloadBuffer;
+    qint64 downloadBufferPosition;
+
     Q_DECLARE_PUBLIC(QNetworkReplyImpl)
 };