QNAM Zerocopy: HTTP implementation part

Reviewed-by: Peter Hartmann
Reviewed-by: Jeremy Katz
Reviewed-by: Aleksandar Sasha Babic

7 files changed, 230 insertions, 67 deletions

diff --git a/src/network/access/qhttpnetworkconnection.cpp b/src/network/access/qhttpnetworkconnection.cpp
index 9e2b85e..400bb12 100644
--- a/src/network/access/qhttpnetworkconnection.cpp
+++ b/src/network/access/qhttpnetworkconnection.cpp
@@ -117,6 +117,7 @@ void QHttpNetworkConnectionPrivate::init()
 {
     for (int i = 0; i < channelCount; i++) {
         channels[i].setConnection(this->q_func());
+        channels[i].ssl = encrypt;
         channels[i].init();
     }
 }
@@ -530,33 +531,35 @@ void QHttpNetworkConnectionPrivate::fillPipeline(QAbstractSocket *socket)
            || channels[i].state == QHttpNetworkConnectionChannel::ReadingState))
         return;
 
-
-    //qDebug() << "QHttpNetworkConnectionPrivate::fillPipeline processing highPriorityQueue, size=" << highPriorityQueue.size() << " alreadyPipelined=" << channels[i].alreadyPipelinedRequests.length();
     int lengthBefore;
     while (!highPriorityQueue.isEmpty()) {
         lengthBefore = channels[i].alreadyPipelinedRequests.length();
         fillPipeline(highPriorityQueue, channels[i]);
 
-        if (channels[i].alreadyPipelinedRequests.length() >= defaultPipelineLength)
+        if (channels[i].alreadyPipelinedRequests.length() >= defaultPipelineLength) {
+            channels[i].pipelineFlush();
             return;
+        }
 
         if (lengthBefore == channels[i].alreadyPipelinedRequests.length())
             break; // did not process anything, now do the low prio queue
     }
 
-    //qDebug() << "QHttpNetworkConnectionPrivate::fillPipeline processing lowPriorityQueue, size=" << lowPriorityQueue.size() << " alreadyPipelined=" << channels[i].alreadyPipelinedRequests.length();
     while (!lowPriorityQueue.isEmpty()) {
         lengthBefore = channels[i].alreadyPipelinedRequests.length();
         fillPipeline(lowPriorityQueue, channels[i]);
 
-        if (channels[i].alreadyPipelinedRequests.length() >= defaultPipelineLength)
+        if (channels[i].alreadyPipelinedRequests.length() >= defaultPipelineLength) {
+            channels[i].pipelineFlush();
             return;
+        }
 
         if (lengthBefore == channels[i].alreadyPipelinedRequests.length())
             break; // did not process anything
     }
 
 
+    channels[i].pipelineFlush();
 }
 
 // returns true when the processing of a queue has been done
diff --git a/src/network/access/qhttpnetworkconnectionchannel.cpp b/src/network/access/qhttpnetworkconnectionchannel.cpp
index 71a4de3..c0b5928 100644
--- a/src/network/access/qhttpnetworkconnectionchannel.cpp
+++ b/src/network/access/qhttpnetworkconnectionchannel.cpp
@@ -60,6 +60,7 @@ QT_BEGIN_NAMESPACE
 
 QHttpNetworkConnectionChannel::QHttpNetworkConnectionChannel()
     : socket(0)
+    , ssl(false)
     , state(IdleState)
     , reply(0)
     , written(0)
@@ -90,11 +91,8 @@ void QHttpNetworkConnectionChannel::init()
 #else
     socket = new QTcpSocket;
 #endif
-
-    // limit the socket read buffer size. we will read everything into
-    // the QHttpNetworkReply anyway, so let's grow only that and not
-    // here and there.
-    socket->setReadBufferSize(64*1024);
+    // Set by QNAM anyway, but let's be safe here
+    socket->setProxy(QNetworkProxy::NoProxy);
 
     QObject::connect(socket, SIGNAL(bytesWritten(qint64)),
                      this, SLOT(_q_bytesWritten(qint64)),
@@ -164,11 +162,12 @@ bool QHttpNetworkConnectionChannel::sendRequest()
         written = 0; // excluding the header
         bytesTotal = 0;
 
-        reply->d_func()->clear();
-        reply->d_func()->connection = connection;
-        reply->d_func()->connectionChannel = this;
-        reply->d_func()->autoDecompress = request.d->autoDecompress;
-        reply->d_func()->pipeliningUsed = false;
+        QHttpNetworkReplyPrivate *replyPrivate = reply->d_func();
+        replyPrivate->clear();
+        replyPrivate->connection = connection;
+        replyPrivate->connectionChannel = this;
+        replyPrivate->autoDecompress = request.d->autoDecompress;
+        replyPrivate->pipeliningUsed = false;
 
         pendingEncrypt = false;
         // if the url contains authentication parameters, use the new ones
@@ -328,7 +327,6 @@ void QHttpNetworkConnectionChannel::_q_receiveReply()
         return;
     }
 
-    qint64 bytes = 0;
     QAbstractSocket::SocketState socketState = socket->state();
 
     // connection might be closed to signal the end of data
@@ -349,12 +347,14 @@ void QHttpNetworkConnectionChannel::_q_receiveReply()
     }
 
     // read loop for the response
-    while (socket->bytesAvailable()) {
+    qint64 bytes = 0;
+    qint64 lastBytes = bytes;
+    do {
+        lastBytes = bytes;
+
         QHttpNetworkReplyPrivate::ReplyState state = reply->d_func()->state;
         switch (state) {
         case QHttpNetworkReplyPrivate::NothingDoneState: {
-            // only eat whitespace on the first call
-            eatWhitespace();
             state = reply->d_func()->state = QHttpNetworkReplyPrivate::ReadingStatusState;
             // fallthrough
         }
@@ -378,6 +378,7 @@ void QHttpNetworkConnectionChannel::_q_receiveReply()
                 return;
             }
             bytes += headerBytes;
+            // If headers were parsed successfully now it is the ReadingDataState
             if (replyPrivate->state == QHttpNetworkReplyPrivate::ReadingDataState) {
                 if (replyPrivate->isGzipped() && replyPrivate->autoDecompress) {
                     // remove the Content-Length from header
@@ -392,6 +393,10 @@ void QHttpNetworkConnectionChannel::_q_receiveReply()
                 }
                 if (replyPrivate->shouldEmitSignals())
                     emit reply->headerChanged();
+                // After headerChanged had been emitted
+                // we can suddenly have a replyPrivate->userProvidedDownloadBuffer
+                // this is handled in the ReadingDataState however
+
                 if (!replyPrivate->expectContent()) {
                     replyPrivate->state = QHttpNetworkReplyPrivate::AllDoneState;
                     allDone();
@@ -412,12 +417,24 @@ void QHttpNetworkConnectionChannel::_q_receiveReply()
                return;
            }
 
-            if (!replyPrivate->isChunked() && !replyPrivate->autoDecompress
-                && replyPrivate->bodyLength > 0) {
-                // bulk files like images should fulfill these properties and
-                // we can therefore save on memory copying
-                bytes = replyPrivate->readBodyFast(socket, &replyPrivate->responseData);
-                replyPrivate->totalProgress += bytes;
+           if (replyPrivate->userProvidedDownloadBuffer) {
+               // the user provided a direct buffer where we should put all our data in.
+               // this only works when we can tell the user the content length and he/she can allocate
+               // the buffer in that size.
+               // note that this call will read only from the still buffered data
+               qint64 haveRead = replyPrivate->readBodyVeryFast(socket, replyPrivate->userProvidedDownloadBuffer + replyPrivate->totalProgress);
+               bytes += haveRead;
+               replyPrivate->totalProgress += haveRead;
+
+               // the user will get notified of it via progress signal
+               emit reply->dataReadProgress(replyPrivate->totalProgress, replyPrivate->bodyLength);
+           } else if (!replyPrivate->isChunked() && !replyPrivate->autoDecompress
+                 && replyPrivate->bodyLength > 0) {
+                 // bulk files like images should fulfill these properties and
+                 // we can therefore save on memory copying
+                qint64 haveRead = replyPrivate->readBodyFast(socket, &replyPrivate->responseData);
+                bytes += haveRead;
+                replyPrivate->totalProgress += haveRead;
                 if (replyPrivate->shouldEmitSignals()) {
                     emit reply->readyRead();
                     emit reply->dataReadProgress(replyPrivate->totalProgress, replyPrivate->bodyLength);
@@ -428,8 +445,9 @@ void QHttpNetworkConnectionChannel::_q_receiveReply()
                 // use the traditional slower reading (for compressed encoding, chunked encoding,
                 // no content-length etc)
                 QByteDataBuffer byteDatas;
-                bytes = replyPrivate->readBody(socket, &byteDatas);
-                if (bytes) {
+                qint64 haveRead = replyPrivate->readBody(socket, &byteDatas);
+                if (haveRead) {
+                    bytes += haveRead;
                     if (replyPrivate->autoDecompress)
                         replyPrivate->appendCompressedReplyData(byteDatas);
                     else
@@ -463,7 +481,7 @@ void QHttpNetworkConnectionChannel::_q_receiveReply()
         default:
             break;
         }
-    }
+    } while (bytes != lastBytes && reply);
 }
 
 // called when unexpectedly reading a -1 or when data is expected but socket is closed
@@ -506,7 +524,7 @@ bool QHttpNetworkConnectionChannel::ensureConnection()
     if (socketState != QAbstractSocket::ConnectedState) {
         // connect to the host if not already connected.
         state = QHttpNetworkConnectionChannel::ConnectingState;
-        pendingEncrypt = connection->d_func()->encrypt;
+        pendingEncrypt = ssl;
 
         // reset state
         pipeliningSupported = PipeliningSupportUnknown;
@@ -529,23 +547,43 @@ bool QHttpNetworkConnectionChannel::ensureConnection()
 
 #ifndef QT_NO_NETWORKPROXY
         // HTTPS always use transparent proxy.
-        if (connection->d_func()->networkProxy.type() != QNetworkProxy::NoProxy && !connection->d_func()->encrypt) {
+        if (connection->d_func()->networkProxy.type() != QNetworkProxy::NoProxy && !ssl) {
             connectHost = connection->d_func()->networkProxy.hostName();
             connectPort = connection->d_func()->networkProxy.port();
         }
 #endif
-        if (connection->d_func()->encrypt) {
+        if (ssl) {
 #ifndef QT_NO_OPENSSL
             QSslSocket *sslSocket = qobject_cast<QSslSocket*>(socket);
             sslSocket->connectToHostEncrypted(connectHost, connectPort);
             if (ignoreAllSslErrors)
                 sslSocket->ignoreSslErrors();
             sslSocket->ignoreSslErrors(ignoreSslErrorsList);
+
+            // limit the socket read buffer size. we will read everything into
+            // the QHttpNetworkReply anyway, so let's grow only that and not
+            // here and there.
+            socket->setReadBufferSize(64*1024);
 #else
             connection->d_func()->emitReplyError(socket, reply, QNetworkReply::ProtocolUnknownError);
 #endif
         } else {
-            socket->connectToHost(connectHost, connectPort);
+            // In case of no proxy we can use the Unbuffered QTcpSocket
+            if (connection->d_func()->networkProxy.type() == QNetworkProxy::NoProxy
+                    && connection->cacheProxy().type() == QNetworkProxy::NoProxy
+                    && connection->transparentProxy().type() == QNetworkProxy::NoProxy) {
+                socket->connectToHost(connectHost, connectPort, QIODevice::ReadWrite | QIODevice::Unbuffered);
+                // For an Unbuffered QTcpSocket, the read buffer size has a special meaning.
+                socket->setReadBufferSize(1*1024);
+
+            } else {
+                socket->connectToHost(connectHost, connectPort);
+
+                // limit the socket read buffer size. we will read everything into
+                // the QHttpNetworkReply anyway, so let's grow only that and not
+                // here and there.
+                socket->setReadBufferSize(64*1024);
+            }
         }
         return false;
     }
@@ -593,6 +631,7 @@ bool QHttpNetworkConnectionChannel::expand(bool dataComplete)
 
 void QHttpNetworkConnectionChannel::allDone()
 {
+    eatWhitespace(); // FIXME HACK!
 #ifndef QT_NO_COMPRESS
     // expand the whole data.
     if (reply->d_func()->expectContent() && reply->d_func()->autoDecompress && !reply->d_func()->streamEnd)
@@ -789,7 +828,7 @@ bool QHttpNetworkConnectionChannel::resetUploadData()
 }
 
 
-void  QHttpNetworkConnectionChannel::pipelineInto(HttpMessagePair &pair)
+void QHttpNetworkConnectionChannel::pipelineInto(HttpMessagePair &pair)
 {
     // this is only called for simple GET
 
@@ -802,16 +841,32 @@ void  QHttpNetworkConnectionChannel::pipelineInto(HttpMessagePair &pair)
     reply->d_func()->pipeliningUsed = true;
 
 #ifndef QT_NO_NETWORKPROXY
-    QByteArray header = QHttpNetworkRequestPrivate::header(request,
-                                                           (connection->d_func()->networkProxy.type() != QNetworkProxy::NoProxy));
+    pipeline.append(QHttpNetworkRequestPrivate::header(request,
+                                                           (connection->d_func()->networkProxy.type() != QNetworkProxy::NoProxy)));
 #else
-    QByteArray header = QHttpNetworkRequestPrivate::header(request, false);
+    pipeline.append(QHttpNetworkRequestPrivate::header(request, false));
 #endif
-    socket->write(header);
 
     alreadyPipelinedRequests.append(pair);
+
+    // pipelineFlush() needs to be called at some point afterwards
 }
 
+void QHttpNetworkConnectionChannel::pipelineFlush()
+{
+    if (pipeline.isEmpty())
+        return;
+
+    // The goal of this is so that we have everything in one TCP packet.
+    // For the Unbuffered QTcpSocket this is manually needed, the buffered
+    // QTcpSocket does it automatically.
+    // Also, sometimes the OS does it for us (Nagle's algorithm) but that
+    // happens only sometimes.
+    socket->write(pipeline);
+    pipeline.clear();
+}
+
+
 void QHttpNetworkConnectionChannel::closeAndResendCurrentRequest()
 {
     requeueCurrentlyPipelinedRequests();
@@ -844,6 +899,17 @@ bool QHttpNetworkConnectionChannel::isSocketReading() const
 //private slots
 void QHttpNetworkConnectionChannel::_q_readyRead()
 {
+    // We got a readyRead but no bytes are available..
+    // This happens for the Unbuffered QTcpSocket
+    if (socket->bytesAvailable() == 0) {
+        char c;
+        qint64  ret = socket->peek(&c, 1);
+        if (ret < 0) {
+            socket->disconnectFromHost();
+            return;
+        }
+    }
+
     if (isSocketWaiting() || isSocketReading()) {
         state = QHttpNetworkConnectionChannel::ReadingState;
         if (reply)
diff --git a/src/network/access/qhttpnetworkconnectionchannel_p.h b/src/network/access/qhttpnetworkconnectionchannel_p.h
index 41a896d..e384e9d 100644
--- a/src/network/access/qhttpnetworkconnectionchannel_p.h
+++ b/src/network/access/qhttpnetworkconnectionchannel_p.h
@@ -99,6 +99,7 @@ public:
         BusyState = (ConnectingState|WritingState|WaitingState|ReadingState|Wait4AuthState)
     };
     QAbstractSocket *socket;
+    bool ssl;
     ChannelState state;
     QHttpNetworkRequest request; // current request
     QHttpNetworkReply *reply; // current reply for this request
@@ -125,7 +126,11 @@ public:
     };
     PipeliningSupport pipeliningSupported;
     QList<HttpMessagePair> alreadyPipelinedRequests;
-
+    QByteArray pipeline; // temporary buffer that gets sent to socket in pipelineFlush
+    void pipelineInto(HttpMessagePair &pair);
+    void pipelineFlush();
+    void requeueCurrentlyPipelinedRequests();
+    void detectPipeliningSupport();
 
     QHttpNetworkConnectionChannel();
     
@@ -145,10 +150,6 @@ public:
 
     bool resetUploadData(); // return true if resetting worked or there is no upload data
 
-    void pipelineInto(HttpMessagePair &pair);
-    void requeueCurrentlyPipelinedRequests();
-    void detectPipeliningSupport();
-
     void handleUnexpectedEOF();
     void closeAndResendCurrentRequest();
 
diff --git a/src/network/access/qhttpnetworkreply.cpp b/src/network/access/qhttpnetworkreply.cpp
index 108ba8a..1c55482 100644
--- a/src/network/access/qhttpnetworkreply.cpp
+++ b/src/network/access/qhttpnetworkreply.cpp
@@ -119,6 +119,7 @@ void QHttpNetworkReply::setRequest(const QHttpNetworkRequest &request)
 {
     Q_D(QHttpNetworkReply);
     d->request = request;
+    d->ssl = request.isSsl();
 }
 
 int QHttpNetworkReply::statusCode() const
@@ -195,6 +196,25 @@ void QHttpNetworkReply::setDownstreamLimited(bool dsl)
     d->connection->d_func()->readMoreLater(this);
 }
 
+bool QHttpNetworkReply::supportsUserProvidedDownloadBuffer()
+{
+    Q_D(QHttpNetworkReply);
+    return (!d->isChunked() && !d->autoDecompress && d->bodyLength > 0);
+}
+
+void QHttpNetworkReply::setUserProvidedDownloadBuffer(char* b)
+{
+    Q_D(QHttpNetworkReply);
+    if (supportsUserProvidedDownloadBuffer())
+        d->userProvidedDownloadBuffer = b;
+}
+
+char* QHttpNetworkReply::userProvidedDownloadBuffer()
+{
+    Q_D(QHttpNetworkReply);
+    return d->userProvidedDownloadBuffer;
+}
+
 bool QHttpNetworkReply::isFinished() const
 {
     return d_func()->state == QHttpNetworkReplyPrivate::AllDoneState;
@@ -207,7 +227,10 @@ bool QHttpNetworkReply::isPipeliningUsed() const
 
 
 QHttpNetworkReplyPrivate::QHttpNetworkReplyPrivate(const QUrl &newUrl)
-    : QHttpNetworkHeaderPrivate(newUrl), state(NothingDoneState), statusCode(100),
+    : QHttpNetworkHeaderPrivate(newUrl)
+    , state(NothingDoneState)
+    , ssl(false)
+    , statusCode(100),
       majorVersion(0), minorVersion(0), bodyLength(0), contentRead(0), totalProgress(0),
       chunkedTransferEncoding(false),
       connectionCloseEnabled(true),
@@ -215,6 +238,7 @@ QHttpNetworkReplyPrivate::QHttpNetworkReplyPrivate(const QUrl &newUrl)
       currentChunkSize(0), currentChunkRead(0), connection(0), initInflate(false),
       autoDecompress(false), responseData(), requestIsPrepared(false)
       ,pipeliningUsed(false), downstreamLimited(false)
+      ,userProvidedDownloadBuffer(0)
 {
 }
 
@@ -448,6 +472,8 @@ qint64 QHttpNetworkReplyPrivate::readStatus(QAbstractSocket *socket)
             return -1; // unexpected EOF
         else if (haveRead == 0)
             break; // read more later
+        else if (haveRead == 1 && bytes == 0 && (c == 11 || c == '\n' || c == '\r' || c == ' ' || c == 31))
+            continue; // Ignore all whitespace that was trailing froma previous request on that socket
 
         bytes++;
 
@@ -623,12 +649,32 @@ bool QHttpNetworkReplyPrivate::isConnectionCloseEnabled()
 
 // note this function can only be used for non-chunked, non-compressed with
 // known content length
+qint64 QHttpNetworkReplyPrivate::readBodyVeryFast(QAbstractSocket *socket, char *b)
+{
+    // This first read is to flush the buffer inside the socket
+    qint64 haveRead = 0;
+    haveRead = socket->read(b, bodyLength - contentRead);
+    if (haveRead == -1) {
+        return 0; // ### error checking here;
+    }
+    contentRead += haveRead;
+
+    if (contentRead == bodyLength) {
+        state = AllDoneState;
+    }
+
+    return haveRead;
+}
+
+// note this function can only be used for non-chunked, non-compressed with
+// known content length
 qint64 QHttpNetworkReplyPrivate::readBodyFast(QAbstractSocket *socket, QByteDataBuffer *rb)
 {
+
     qint64 toBeRead = qMin(socket->bytesAvailable(), bodyLength - contentRead);
     QByteArray bd;
     bd.resize(toBeRead);
-    qint64 haveRead = socket->read(bd.data(), bd.size());
+    qint64 haveRead = socket->read(bd.data(), toBeRead);
     if (haveRead == -1) {
         bd.clear();
         return 0; // ### error checking here;
@@ -650,29 +696,34 @@ qint64 QHttpNetworkReplyPrivate::readBody(QAbstractSocket *socket, QByteDataBuff
 {
     qint64 bytes = 0;
     if (isChunked()) {
-        bytes += readReplyBodyChunked(socket, out); // chunked transfer encoding (rfc 2616, sec 3.6)
-    } else if (bodyLength > 0) { // we have a Content-Length
+        // chunked transfer encoding (rfc 2616, sec 3.6)
+        bytes += readReplyBodyChunked(socket, out);
+    } else if (bodyLength > 0) {
+        // we have a Content-Length
         bytes += readReplyBodyRaw(socket, out, bodyLength - contentRead);
         if (contentRead + bytes == bodyLength)
             state = AllDoneState;
     } else {
+        // no content length. just read what's possible
         bytes += readReplyBodyRaw(socket, out, socket->bytesAvailable());
     }
     contentRead += bytes;
     return bytes;
 }
 
-qint64 QHttpNetworkReplyPrivate::readReplyBodyRaw(QIODevice *in, QByteDataBuffer *out, qint64 size)
+qint64 QHttpNetworkReplyPrivate::readReplyBodyRaw(QAbstractSocket *socket, QByteDataBuffer *out, qint64 size)
 {
+    // FIXME get rid of this function and just use readBodyFast and give it socket->bytesAvailable()
     qint64 bytes = 0;
-    Q_ASSERT(in);
+    Q_ASSERT(socket);
     Q_ASSERT(out);
 
-    int toBeRead = qMin<qint64>(128*1024, qMin<qint64>(size, in->bytesAvailable()));
+    int toBeRead = qMin<qint64>(128*1024, qMin<qint64>(size, socket->bytesAvailable()));
+
     while (toBeRead > 0) {
         QByteArray byteData;
         byteData.resize(toBeRead);
-        qint64 haveRead = in->read(byteData.data(), byteData.size());
+        qint64 haveRead = socket->read(byteData.data(), byteData.size());
         if (haveRead <= 0) {
             // ### error checking here
             byteData.clear();
@@ -684,25 +735,35 @@ qint64 QHttpNetworkReplyPrivate::readReplyBodyRaw(QIODevice *in, QByteDataBuffer
         bytes += haveRead;
         size -= haveRead;
 
-        toBeRead = qMin<qint64>(128*1024, qMin<qint64>(size, in->bytesAvailable()));
+        toBeRead = qMin<qint64>(128*1024, qMin<qint64>(size, socket->bytesAvailable()));
     }
     return bytes;
 
 }
 
-qint64 QHttpNetworkReplyPrivate::readReplyBodyChunked(QIODevice *in, QByteDataBuffer *out)
+qint64 QHttpNetworkReplyPrivate::readReplyBodyChunked(QAbstractSocket *socket, QByteDataBuffer *out)
 {
     qint64 bytes = 0;
-    while (in->bytesAvailable()) { // while we can read from input
-        // if we are done with the current chunk, get the size of the new chunk
+    while (socket->bytesAvailable()) {
         if (currentChunkRead >= currentChunkSize) {
+            // For the first chunk and when we're done with a chunk
             currentChunkSize = 0;
             currentChunkRead = 0;
             if (bytes) {
+                // After a chunk
                 char crlf[2];
-                bytes += in->read(crlf, 2); // read the "\r\n" after the chunk
+                // read the "\r\n" after the chunk
+                qint64 haveRead = socket->read(crlf, 2);
+                // FIXME: This code is slightly broken and not optimal. What if the 2 bytes are not available yet?!
+                // For nice reasons (the toLong in getChunkSize accepting \n at the beginning
+                // it right now still works, but we should definitely fix this.
+
+                if (haveRead != 2)
+                    return bytes; // FIXME
+                bytes += haveRead;
             }
-            bytes += getChunkSize(in, &currentChunkSize);
+            // Note that chunk size gets stored in currentChunkSize, what is returned is the bytes read
+            bytes += getChunkSize(socket, &currentChunkSize);
             if (currentChunkSize == -1)
                 break;
         }
@@ -712,8 +773,8 @@ qint64 QHttpNetworkReplyPrivate::readReplyBodyChunked(QIODevice *in, QByteDataBu
             break;
         }
 
-        // otherwise, try to read what is missing for this chunk
-        qint64 haveRead = readReplyBodyRaw (in, out, currentChunkSize - currentChunkRead);
+        // otherwise, try to begin reading this chunk / to read what is missing for this chunk
+        qint64 haveRead = readReplyBodyRaw (socket, out, currentChunkSize - currentChunkRead);
         currentChunkRead += haveRead;
         bytes += haveRead;
 
@@ -723,22 +784,25 @@ qint64 QHttpNetworkReplyPrivate::readReplyBodyChunked(QIODevice *in, QByteDataBu
     return bytes;
 }
 
-qint64 QHttpNetworkReplyPrivate::getChunkSize(QIODevice *in, qint64 *chunkSize)
+qint64 QHttpNetworkReplyPrivate::getChunkSize(QAbstractSocket *socket, qint64 *chunkSize)
 {
     qint64 bytes = 0;
     char crlf[2];
     *chunkSize = -1;
-    int bytesAvailable = in->bytesAvailable();
+
+    int bytesAvailable = socket->bytesAvailable();
+    // FIXME rewrite to permanent loop without bytesAvailable
     while (bytesAvailable > bytes) {
-        qint64 sniffedBytes =  in->peek(crlf, 2);
+        qint64 sniffedBytes = socket->peek(crlf, 2);
         int fragmentSize = fragment.size();
+
         // check the next two bytes for a "\r\n", skip blank lines
         if ((fragmentSize && sniffedBytes == 2 && crlf[0] == '\r' && crlf[1] == '\n')
            ||(fragmentSize > 1 && fragment.endsWith('\r')  && crlf[0] == '\n'))
         {
-            bytes += in->read(crlf, 1);     // read the \r or \n
+            bytes += socket->read(crlf, 1);     // read the \r or \n
             if (crlf[0] == '\r')
-                bytes += in->read(crlf, 1); // read the \n
+                bytes += socket->read(crlf, 1); // read the \n
             bool ok = false;
             // ignore the chunk-extension
             fragment = fragment.mid(0, fragment.indexOf(';')).trimmed();
@@ -748,10 +812,15 @@ qint64 QHttpNetworkReplyPrivate::getChunkSize(QIODevice *in, qint64 *chunkSize)
         } else {
             // read the fragment to the buffer
             char c = 0;
-            bytes += in->read(&c, 1);
+            qint64 haveRead = socket->read(&c, 1);
+            if (haveRead < 0) {
+                return -1; // FIXME
+            }
+            bytes += haveRead;
             fragment.append(c);
         }
     }
+
     return bytes;
 }
 
diff --git a/src/network/access/qhttpnetworkreply_p.h b/src/network/access/qhttpnetworkreply_p.h
index 4011c78..02ce248 100644
--- a/src/network/access/qhttpnetworkreply_p.h
+++ b/src/network/access/qhttpnetworkreply_p.h
@@ -128,6 +128,10 @@ public:
     QByteArray readAny();
     void setDownstreamLimited(bool t);
 
+    bool supportsUserProvidedDownloadBuffer();
+    void setUserProvidedDownloadBuffer(char*);
+    char* userProvidedDownloadBuffer();
+
     bool isFinished() const;
 
     bool isPipeliningUsed() const;
@@ -147,6 +151,7 @@ Q_SIGNALS:
     void finished();
     void finishedWithError(QNetworkReply::NetworkError errorCode, const QString &detail = QString());
     void headerChanged();
+    // FIXME we need to change this to qint64!
     void dataReadProgress(int done, int total);
     void dataSendProgress(qint64 done, qint64 total);
 
@@ -168,15 +173,16 @@ public:
     qint64 readHeader(QAbstractSocket *socket);
     void parseHeader(const QByteArray &header);
     qint64 readBody(QAbstractSocket *socket, QByteDataBuffer *out);
+    qint64 readBodyVeryFast(QAbstractSocket *socket, char *b);
     qint64 readBodyFast(QAbstractSocket *socket, QByteDataBuffer *rb);
     bool findChallenge(bool forProxy, QByteArray &challenge) const;
     QAuthenticatorPrivate::Method authenticationMethod(bool isProxy) const;
     void clear();
     void clearHttpLayerInformation();
 
-    qint64 readReplyBodyRaw(QIODevice *in, QByteDataBuffer *out, qint64 size);
-    qint64 readReplyBodyChunked(QIODevice *in, QByteDataBuffer *out);
-    qint64 getChunkSize(QIODevice *in, qint64 *chunkSize);
+    qint64 readReplyBodyRaw(QAbstractSocket *in, QByteDataBuffer *out, qint64 size);
+    qint64 readReplyBodyChunked(QAbstractSocket *in, QByteDataBuffer *out);
+    qint64 getChunkSize(QAbstractSocket *in, qint64 *chunkSize);
 
     void appendUncompressedReplyData(QByteArray &qba);
     void appendUncompressedReplyData(QByteDataBuffer &data);
@@ -205,6 +211,7 @@ public:
     } state;
 
     QHttpNetworkRequest request;
+    bool ssl;
     int statusCode;
     int majorVersion;
     int minorVersion;
@@ -234,6 +241,8 @@ public:
 
     bool pipeliningUsed;
     bool downstreamLimited;
+
+    char* userProvidedDownloadBuffer;
 };
 
 
diff --git a/src/network/access/qhttpnetworkrequest.cpp b/src/network/access/qhttpnetworkrequest.cpp
index 639025e..d2f3212 100644
--- a/src/network/access/qhttpnetworkrequest.cpp
+++ b/src/network/access/qhttpnetworkrequest.cpp
@@ -63,6 +63,7 @@ QHttpNetworkRequestPrivate::QHttpNetworkRequestPrivate(const QHttpNetworkRequest
     pipeliningAllowed = other.pipeliningAllowed;
     customVerb = other.customVerb;
     withCredentials = other.withCredentials;
+    ssl = other.ssl;
 }
 
 QHttpNetworkRequestPrivate::~QHttpNetworkRequestPrivate()
@@ -73,6 +74,7 @@ bool QHttpNetworkRequestPrivate::operator==(const QHttpNetworkRequestPrivate &ot
 {
     return QHttpNetworkHeaderPrivate::operator==(other)
         && (operation == other.operation)
+        && (ssl == other.ssl)
         && (uploadByteDevice == other.uploadByteDevice);
 }
 
@@ -199,6 +201,15 @@ void QHttpNetworkRequest::setUrl(const QUrl &url)
     d->url = url;
 }
 
+bool QHttpNetworkRequest::isSsl() const
+{
+    return d->ssl;
+}
+void QHttpNetworkRequest::setSsl(bool s)
+{
+    d->ssl = s;
+}
+
 qint64 QHttpNetworkRequest::contentLength() const
 {
     return d->contentLength();
diff --git a/src/network/access/qhttpnetworkrequest_p.h b/src/network/access/qhttpnetworkrequest_p.h
index 15cab73..123babc 100644
--- a/src/network/access/qhttpnetworkrequest_p.h
+++ b/src/network/access/qhttpnetworkrequest_p.h
@@ -116,6 +116,9 @@ public:
     bool withCredentials() const;
     void setWithCredentials(bool b);
 
+    bool isSsl() const;
+    void setSsl(bool);
+
     void setUploadByteDevice(QNonContiguousByteDevice *bd);
     QNonContiguousByteDevice* uploadByteDevice() const;
 
@@ -146,6 +149,7 @@ public:
     bool autoDecompress;
     bool pipeliningAllowed;
     bool withCredentials;
+    bool ssl;
 };