QNAM HTTP Pipelining

HTTP Pipelining should improve the performance of HTTP requests
for high latency network links. Since some servers/proxies could
have problems with it, it is disabled by default.
Set the HttpPipeliningAllowed attribute of a QNetworkRequest
to enable it for that request.

Reviewed-by: Thiago

14 files changed, 519 insertions, 52 deletions

diff --git a/demos/browser/networkaccessmanager.h b/demos/browser/networkaccessmanager.h
index 381cb50..4c4603e 100644
--- a/demos/browser/networkaccessmanager.h
+++ b/demos/browser/networkaccessmanager.h
@@ -43,6 +43,7 @@
 #define NETWORKACCESSMANAGER_H
 
 #include <QtNetwork/QNetworkAccessManager>
+#include <QtNetwork/QNetworkRequest>
 
 class NetworkAccessManager : public QNetworkAccessManager
 {
@@ -51,6 +52,14 @@ class NetworkAccessManager : public QNetworkAccessManager
 public:
     NetworkAccessManager(QObject *parent = 0);
 
+    // this is a temporary hack until we properly use the pipelining flags from QtWebkit
+    // pipeline everything! :)
+    virtual QNetworkReply* createRequest ( Operation op, const QNetworkRequest & req, QIODevice * outgoingData = 0 ) {
+        QNetworkRequest request = req; // copy so we can modify
+        request.setAttribute(QNetworkRequest::HttpPipeliningAllowedAttribute, true);
+        return QNetworkAccessManager::createRequest(op, request, outgoingData);
+    }
+
 private:
     QList<QString> sslTrustedHostList;
 
diff --git a/src/network/access/qhttpnetworkconnection.cpp b/src/network/access/qhttpnetworkconnection.cpp
index 6888891..397e0b5 100644
--- a/src/network/access/qhttpnetworkconnection.cpp
+++ b/src/network/access/qhttpnetworkconnection.cpp
@@ -67,6 +67,11 @@ QT_BEGIN_NAMESPACE
 
 const int QHttpNetworkConnectionPrivate::defaultChannelCount = 6;
 
+// the maximum amount of requests that might be pipelined into a socket
+// from what was suggested, 3 seems to be OK
+const int QHttpNetworkConnectionPrivate::defaultPipelineLength = 3;
+
+
 QHttpNetworkConnectionPrivate::QHttpNetworkConnectionPrivate(const QString &hostName, quint16 port, bool encrypt)
 : hostName(hostName), port(port), encrypt(encrypt),
   channelCount(defaultChannelCount),
@@ -414,7 +419,24 @@ QHttpNetworkReply* QHttpNetworkConnectionPrivate::queueRequest(const QHttpNetwor
     return reply;
 }
 
-void QHttpNetworkConnectionPrivate::unqueueAndSendRequest(QAbstractSocket *socket)
+void QHttpNetworkConnectionPrivate::requeueRequest(const HttpMessagePair &pair)
+{
+    Q_Q(QHttpNetworkConnection);
+
+    QHttpNetworkRequest request = pair.first;
+    switch (request.priority()) {
+    case QHttpNetworkRequest::HighPriority:
+        highPriorityQueue.prepend(pair);
+        break;
+    case QHttpNetworkRequest::NormalPriority:
+    case QHttpNetworkRequest::LowPriority:
+        lowPriorityQueue.prepend(pair);
+        break;
+    }
+    QMetaObject::invokeMethod(q, "_q_startNextRequest", Qt::QueuedConnection);
+}
+
+void QHttpNetworkConnectionPrivate::dequeueAndSendRequest(QAbstractSocket *socket)
 {
     Q_ASSERT(socket);
 
@@ -428,8 +450,9 @@ void QHttpNetworkConnectionPrivate::unqueueAndSendRequest(QAbstractSocket *socke
 
             channels[i].request = messagePair.first;
             channels[i].reply = messagePair.second;
-            channels[i].sendRequest();
+            // remove before sendRequest! else we might pipeline the same request again
             highPriorityQueue.removeAt(j);
+            channels[i].sendRequest();
             return;
         }
     }
@@ -441,23 +464,113 @@ void QHttpNetworkConnectionPrivate::unqueueAndSendRequest(QAbstractSocket *socke
                 prepareRequest(messagePair);
             channels[i].request = messagePair.first;
             channels[i].reply = messagePair.second;
-            channels[i].sendRequest();
+            // remove before sendRequest! else we might pipeline the same request again
             lowPriorityQueue.removeAt(j);
+            channels[i].sendRequest();
             return;
         }
     }
 }
 
-void QHttpNetworkConnectionPrivate::resendCurrentRequest(QAbstractSocket *socket)
+// this is called from _q_startNextRequest and when a request has been sent down a socket from the channel
+void QHttpNetworkConnectionPrivate::fillPipeline(QAbstractSocket *socket)
 {
-    Q_Q(QHttpNetworkConnection);
-    Q_ASSERT(socket);
+    // return fast if there is nothing to pipeline
+    if (highPriorityQueue.isEmpty() && lowPriorityQueue.isEmpty())
+        return;
+
     int i = indexOf(socket);
-    channels[i].close();
-    channels[i].resendCurrent = true;
-    QMetaObject::invokeMethod(q, "_q_startNextRequest", Qt::QueuedConnection);
+
+    bool highPriorityQueueProcessingDone = false;
+    bool lowPriorityQueueProcessingDone = false;
+
+    while (!highPriorityQueueProcessingDone && !lowPriorityQueueProcessingDone) {
+        // this loop runs once per request we intend to pipeline in.
+
+        if (channels[i].pipeliningSupported != QHttpNetworkConnectionChannel::PipeliningProbablySupported)
+            return;
+
+        // the current request that is in must already support pipelining
+        if (!channels[i].request.isPipeliningAllowed())
+            return;
+
+        // the current request must be a idempotent (right now we only check GET)
+        if (channels[i].request.operation() != QHttpNetworkRequest::Get)
+            return;
+
+        // check if socket is connected
+        if (socket->state() != QAbstractSocket::ConnectedState)
+            return;
+
+        // check for resendCurrent
+        if (channels[i].resendCurrent)
+            return;
+
+        // we do not like authentication stuff
+        // ### make sure to be OK with this in later releases
+        if (!channels[i].authenticator.isNull() || !channels[i].authenticator.user().isEmpty())
+            return;
+        if (!channels[i].proxyAuthenticator.isNull() || !channels[i].proxyAuthenticator.user().isEmpty())
+            return;
+
+        // check for pipeline length
+        if (channels[i].alreadyPipelinedRequests.length() >= defaultPipelineLength)
+            return;
+
+        // must be in ReadingState or WaitingState
+        if (! (channels[i].state == QHttpNetworkConnectionChannel::WaitingState
+               || channels[i].state == QHttpNetworkConnectionChannel::ReadingState))
+            return;
+
+        highPriorityQueueProcessingDone = fillPipeline(highPriorityQueue, channels[i]);
+        // not finished with highPriorityQueue? then loop again
+        if (!highPriorityQueueProcessingDone)
+            continue;
+        // highPriorityQueue was processed, now deal with the lowPriorityQueue
+        lowPriorityQueueProcessingDone = fillPipeline(lowPriorityQueue, channels[i]);
+    }
+}
+
+// returns true when the processing of a queue has been done
+bool QHttpNetworkConnectionPrivate::fillPipeline(QList<HttpMessagePair> &queue, QHttpNetworkConnectionChannel &channel)
+{
+    if (queue.isEmpty())
+        return true;
+
+    for (int i = 0; i < queue.length(); i++) {
+        HttpMessagePair messagePair = queue.at(i);
+        const QHttpNetworkRequest &request = messagePair.first;
+
+        // we currently do not support pipelining if HTTP authentication is used
+        if (!request.url().userInfo().isEmpty())
+            continue;
+
+        // take only GET requests
+        if (request.operation() != QHttpNetworkRequest::Get)
+            continue;
+
+        if (!request.isPipeliningAllowed())
+            continue;
+
+        // remove it from the queue
+        queue.takeAt(i);
+        // we modify the queue we iterate over here, but since we return from the function
+        // afterwards this is fine.
+
+        // actually send it
+        if (!messagePair.second->d_func()->requestIsPrepared)
+            prepareRequest(messagePair);
+        channel.pipelineInto(messagePair);
+
+        // return false because we processed something and need to process again
+        return false;
+    }
+
+    // return true, the queue has been processed and not changed
+    return true;
 }
 
+
 QString QHttpNetworkConnectionPrivate::errorDetail(QNetworkReply::NetworkError errorCode, QAbstractSocket* socket)
 {
     Q_ASSERT(socket);
@@ -560,16 +673,29 @@ void QHttpNetworkConnectionPrivate::_q_startNextRequest()
             break;
         }
     }
-    if (!socket)
-        return; // this will be called after finishing current request.
-    unqueueAndSendRequest(socket);
+
+    // this socket is free,
+    if (socket)
+        dequeueAndSendRequest(socket);
+
+    // try to push more into all sockets
+    // ### FIXME we should move this to the beginning of the function
+    // as soon as QtWebkit is properly using the pipelining
+    // (e.g. not for XMLHttpRequest or the first page load)
+    // ### FIXME we should also divide the requests more even
+    // on the connected sockets
+    //tryToFillPipeline(socket);
+    // return fast if there is nothing to pipeline
+    if (highPriorityQueue.isEmpty() && lowPriorityQueue.isEmpty())
+        return;
+    for (int j = 0; j < channelCount; j++)
+        fillPipeline(channels[j].socket);
 }
 
 void QHttpNetworkConnectionPrivate::_q_restartAuthPendingRequests()
 {
     // send the request using the idle socket
     for (int i = 0 ; i < channelCount; ++i) {
-        QAbstractSocket *socket = channels[i].socket;
         if (channels[i].state ==  QHttpNetworkConnectionChannel::Wait4AuthState) {
             channels[i].state = QHttpNetworkConnectionChannel::IdleState;
             if (channels[i].reply)
@@ -739,6 +865,19 @@ void QHttpNetworkConnection::ignoreSslErrors(const QList<QSslError> &errors, int
 
 #endif //QT_NO_OPENSSL
 
+#ifndef QT_NO_NETWORKPROXY
+// only called from QHttpNetworkConnectionChannel::_q_proxyAuthenticationRequired, not
+// from QHttpNetworkConnectionChannel::handleAuthenticationChallenge
+// e.g. it is for SOCKS proxies which require authentication.
+void QHttpNetworkConnectionPrivate::emitProxyAuthenticationRequired(const QHttpNetworkConnectionChannel *chan, const QNetworkProxy &proxy, QAuthenticator* auth)
+{
+    Q_Q(QHttpNetworkConnection);
+    emit q->proxyAuthenticationRequired(proxy, auth, q);
+    int i = indexOf(chan->socket);
+    copyCredentials(i, auth, true);
+}
+#endif
+
 
 QT_END_NAMESPACE
 
diff --git a/src/network/access/qhttpnetworkconnection_p.h b/src/network/access/qhttpnetworkconnection_p.h
index 3f928ec..af764ed 100644
--- a/src/network/access/qhttpnetworkconnection_p.h
+++ b/src/network/access/qhttpnetworkconnection_p.h
@@ -155,6 +155,8 @@ class QHttpNetworkConnectionPrivate : public QObjectPrivate
     Q_DECLARE_PUBLIC(QHttpNetworkConnection)
 public:
     static const int defaultChannelCount;
+    static const int defaultPipelineLength;
+
     QHttpNetworkConnectionPrivate(const QString &hostName, quint16 port, bool encrypt);
     QHttpNetworkConnectionPrivate(quint16 channelCount, const QString &hostName, quint16 port, bool encrypt);
     ~QHttpNetworkConnectionPrivate();
@@ -169,9 +171,12 @@ public:
     bool isSocketReading(QAbstractSocket *socket) const;
 
     QHttpNetworkReply *queueRequest(const QHttpNetworkRequest &request);
-    void unqueueAndSendRequest(QAbstractSocket *socket);
+    void requeueRequest(const HttpMessagePair &pair); // e.g. after pipeline broke
+    void dequeueAndSendRequest(QAbstractSocket *socket);
     void prepareRequest(HttpMessagePair &request);
-    void resendCurrentRequest(QAbstractSocket *socket);
+
+    void fillPipeline(QAbstractSocket *socket);
+    bool fillPipeline(QList<HttpMessagePair> &queue, QHttpNetworkConnectionChannel &channel);
 
     void copyCredentials(int fromChannel, QAuthenticator *auth, bool isProxy);
 
@@ -212,6 +217,7 @@ public:
 
 #ifndef QT_NO_NETWORKPROXY
     QNetworkProxy networkProxy;
+    void emitProxyAuthenticationRequired(const QHttpNetworkConnectionChannel *chan, const QNetworkProxy &proxy, QAuthenticator* auth);
 #endif
 
     //The request queues
diff --git a/src/network/access/qhttpnetworkconnectionchannel.cpp b/src/network/access/qhttpnetworkconnectionchannel.cpp
index 2d33527..221c27c 100644
--- a/src/network/access/qhttpnetworkconnectionchannel.cpp
+++ b/src/network/access/qhttpnetworkconnectionchannel.cpp
@@ -165,8 +165,8 @@ bool QHttpNetworkConnectionChannel::sendRequest()
 
             bytesTotal = request.contentLength();
         } else {
-            socket->flush(); // ### Remove this when pipelining is implemented. We want less TCP packets!
             state = QHttpNetworkConnectionChannel::WaitingState;
+            sendRequest();
             break;
         }
         // write the initial chunk together with the headers
@@ -242,6 +242,11 @@ bool QHttpNetworkConnectionChannel::sendRequest()
         if (uploadByteDevice) {
             QObject::disconnect(uploadByteDevice, SIGNAL(readyRead()), this, SLOT(_q_uploadDataReadyRead()));
         }
+
+        // HTTP pipelining
+        connection->d_func()->fillPipeline(socket);
+        socket->flush();
+
         // ensure we try to receive a reply in all cases, even if _q_readyRead_ hat not been called
         // this is needed if the sends an reply before we have finished sending the request. In that
         // case receiveReply had been called before but ignored the server reply
@@ -276,7 +281,7 @@ void QHttpNetworkConnectionChannel::receiveReply()
             } else {
                 // try to reconnect/resend before sending an error.
                 if (reconnectAttempts-- > 0) {
-                    connection->d_func()->resendCurrentRequest(socket);
+                    closeAndResendCurrentRequest();
                 } else if (reply) {
                     reply->d_func()->errorString = connection->d_func()->errorDetail(QNetworkReply::RemoteHostClosedError, socket);
                     emit reply->finishedWithError(QNetworkReply::RemoteHostClosedError, reply->d_func()->errorString);
@@ -292,14 +297,20 @@ void QHttpNetworkConnectionChannel::receiveReply()
         switch (state) {
         case QHttpNetworkReplyPrivate::NothingDoneState:
         case QHttpNetworkReplyPrivate::ReadingStatusState: {
+            eatWhitespace();
             qint64 statusBytes = reply->d_func()->readStatus(socket);
-            if (statusBytes == -1) {
-                // error reading the status, close the socket and emit error
-                socket->close();
+            if (statusBytes == -1 && reconnectAttempts <= 0) {
+                // too many errors reading/receiving/parsing the status, close the socket and emit error
+                close();
                 reply->d_func()->errorString = connection->d_func()->errorDetail(QNetworkReply::ProtocolFailure, socket);
                 emit reply->finishedWithError(QNetworkReply::ProtocolFailure, reply->d_func()->errorString);
                 QMetaObject::invokeMethod(connection, "_q_startNextRequest", Qt::QueuedConnection);
                 break;
+            } else if (statusBytes == -1) {
+                reconnectAttempts--;
+                reply->d_func()->clear();
+                closeAndResendCurrentRequest();
+                break;
             }
             bytes += statusBytes;
             lastStatus = reply->d_func()->statusCode;
@@ -410,6 +421,9 @@ bool QHttpNetworkConnectionChannel::ensureConnection()
         state = QHttpNetworkConnectionChannel::ConnectingState;
         pendingEncrypt = connection->d_func()->encrypt;
 
+        // reset state
+        pipeliningSupported = PipeliningSupportUnknown;
+
         // This workaround is needed since we use QAuthenticator for NTLM authentication. The "phase == Done"
         // is the usual criteria for emitting authentication signals. The "phase" is set to "Done" when the
         // last header for Authorization is generated by the QAuthenticator. Basic & Digest logic does not
@@ -520,6 +534,92 @@ void QHttpNetworkConnectionChannel::allDone()
     // reset the reconnection attempts after we receive a complete reply.
     // in case of failures, each channel will attempt two reconnects before emitting error.
     reconnectAttempts = 2;
+
+    detectPipeliningSupport();
+
+    // move next from pipeline to current request
+    if (!alreadyPipelinedRequests.isEmpty()) {
+        if (resendCurrent || reply->d_func()->connectionCloseEnabled() || socket->state() != QAbstractSocket::Connected) {
+            // move the pipelined ones back to the main queue
+            requeueCurrentlyPipelinedRequests();
+        } else {
+            // there were requests pipelined in and we can continue
+            HttpMessagePair messagePair = alreadyPipelinedRequests.takeFirst();
+
+            request = messagePair.first;
+            reply = messagePair.second;
+            state = QHttpNetworkConnectionChannel::ReadingState;
+            resendCurrent = false;
+
+            written = 0; // message body, excluding the header, irrelevant here
+            bytesTotal = 0; // message body total, excluding the header, irrelevant here
+
+            // pipeline even more
+            connection->d_func()->fillPipeline(socket);
+
+            // continue reading
+            receiveReply();
+        }
+    } else if (alreadyPipelinedRequests.isEmpty() && socket->bytesAvailable() > 0) {
+        eatWhitespace();
+        // this is weird. we had nothing pipelined but still bytes available. better close it.
+        if (socket->bytesAvailable() > 0)
+            close();
+        QMetaObject::invokeMethod(connection, "_q_startNextRequest", Qt::QueuedConnection);
+    } else if (alreadyPipelinedRequests.isEmpty()) {
+        QMetaObject::invokeMethod(connection, "_q_startNextRequest", Qt::QueuedConnection);
+    }
+}
+
+void QHttpNetworkConnectionChannel::detectPipeliningSupport()
+{
+    // detect HTTP Pipelining support
+    QByteArray serverHeaderField = reply->headerField("Server");
+    if (
+            // check for broken servers in server reply header
+            // this is adapted from http://mxr.mozilla.org/firefox/ident?i=SupportsPipelining
+            (!serverHeaderField.contains("Microsoft-IIS/4."))
+            && (!serverHeaderField.contains("Microsoft-IIS/5."))
+            && (!serverHeaderField.contains("Netscape-Enterprise/3."))
+            // check for HTTP/1.1
+            && (reply->d_func()->majorVersion == 1 && reply->d_func()->minorVersion == 1)
+            // check for not having connection close
+            && (!reply->d_func()->connectionCloseEnabled())
+            // check if it is still connected
+            && (socket->state() == QAbstractSocket::Connected)
+            ) {
+        pipeliningSupported = QHttpNetworkConnectionChannel::PipeliningProbablySupported;
+    } else {
+        pipeliningSupported = QHttpNetworkConnectionChannel::PipeliningSupportUnknown;
+    }
+}
+
+// called when the connection broke and we need to queue some pipelined requests again
+void QHttpNetworkConnectionChannel::requeueCurrentlyPipelinedRequests()
+{
+    for (int i = 0; i < alreadyPipelinedRequests.length(); i++)
+        connection->d_func()->requeueRequest(alreadyPipelinedRequests.at(i));
+    alreadyPipelinedRequests.clear();
+
+    close();
+    QMetaObject::invokeMethod(connection, "_q_startNextRequest", Qt::QueuedConnection);
+}
+
+void QHttpNetworkConnectionChannel::eatWhitespace()
+{
+    char c;
+    while (socket->bytesAvailable()) {
+        if (socket->peek(&c, 1) != 1)
+            return;
+
+        // read all whitespace and line endings
+        if (c == 11 || c == '\n' || c == '\r' || c == ' ' || c == 31) {
+            socket->read(&c, 1);
+            continue;
+        } else {
+            break;
+        }
+    }
 }
 
 void QHttpNetworkConnectionChannel::handleStatus()
@@ -531,8 +631,8 @@ void QHttpNetworkConnectionChannel::handleStatus()
     bool resend = false;
 
     switch (statusCode) {
-    case 401:
-    case 407:
+    case 401: // auth required
+    case 407: // proxy auth required
         if (connection->d_func()->handleAuthenticateChallenge(socket, reply, (statusCode == 407), resend)) {
             if (resend) {
                 QNonContiguousByteDevice* uploadByteDevice = request.uploadByteDevice();
@@ -547,10 +647,15 @@ void QHttpNetworkConnectionChannel::handleStatus()
 
                 reply->d_func()->eraseData();
 
-                // also use async _q_startNextRequest so we dont break with closed
-                // proxy or server connections..
-                resendCurrent = true;
-                QMetaObject::invokeMethod(connection, "_q_startNextRequest", Qt::QueuedConnection);
+                if (alreadyPipelinedRequests.isEmpty()) {
+                    // this does a re-send without closing the connection
+                    resendCurrent = true;
+                    QMetaObject::invokeMethod(connection, "_q_startNextRequest", Qt::QueuedConnection);
+                } else {
+                    // we had requests pipelined.. better close the connection in closeAndResendCurrentRequest
+                    closeAndResendCurrentRequest();
+                    QMetaObject::invokeMethod(connection, "_q_startNextRequest", Qt::QueuedConnection);
+                }
             }
         } else {
             emit reply->headerChanged();
@@ -568,26 +673,51 @@ void QHttpNetworkConnectionChannel::handleStatus()
     }
 }
 
+void  QHttpNetworkConnectionChannel::pipelineInto(HttpMessagePair &pair)
+{
+    // this is only called for simple GET
+
+    QHttpNetworkRequest &request = pair.first;
+    QHttpNetworkReply *reply = pair.second;
+    if (reply) {
+        reply->d_func()->clear();
+        reply->d_func()->connection = connection;
+        reply->d_func()->autoDecompress = request.d->autoDecompress;
+    }
 
+#ifndef QT_NO_NETWORKPROXY
+    QByteArray header = QHttpNetworkRequestPrivate::header(request,
+                                                           (connection->d_func()->networkProxy.type() != QNetworkProxy::NoProxy));
+#else
+    QByteArray header = QHttpNetworkRequestPrivate::header(channels[i].request,
+                                                           false);
+#endif
+    socket->write(header);
+
+    alreadyPipelinedRequests.append(pair);
+}
+
+void QHttpNetworkConnectionChannel::closeAndResendCurrentRequest()
+{
+    requeueCurrentlyPipelinedRequests();
+    close();
+    resendCurrent = true;
+    QMetaObject::invokeMethod(connection, "_q_startNextRequest", Qt::QueuedConnection);
+}
 
 //private slots
 void QHttpNetworkConnectionChannel::_q_readyRead()
 {
-    if (!socket)
-        return; // ### error
     if (connection->d_func()->isSocketWaiting(socket) || connection->d_func()->isSocketReading(socket)) {
         state = QHttpNetworkConnectionChannel::ReadingState;
         if (reply)
             receiveReply();
     }
-    // ### error
 }
 
 void QHttpNetworkConnectionChannel::_q_bytesWritten(qint64 bytes)
 {
     Q_UNUSED(bytes);
-    if (!socket)
-        return; // ### error
     // bytes have been written to the socket. write even more of them :)
     if (connection->d_func()->isSocketWriting(socket))
         sendRequest();
@@ -596,8 +726,6 @@ void QHttpNetworkConnectionChannel::_q_bytesWritten(qint64 bytes)
 
 void QHttpNetworkConnectionChannel::_q_disconnected()
 {
-    if (!socket)
-        return; // ### error
     // read the available data before closing
     if (connection->d_func()->isSocketWaiting(socket) || connection->d_func()->isSocketReading(socket)) {
         state = QHttpNetworkConnectionChannel::ReadingState;
@@ -608,17 +736,18 @@ void QHttpNetworkConnectionChannel::_q_disconnected()
         QMetaObject::invokeMethod(connection, "_q_startNextRequest", Qt::QueuedConnection);
     }
     state = QHttpNetworkConnectionChannel::IdleState;
+
+    requeueCurrentlyPipelinedRequests();
 }
 
 
 void QHttpNetworkConnectionChannel::_q_connected()
 {
-    if (!socket)
-        return; // ### error
-
     // improve performance since we get the request sent by the kernel ASAP
     socket->setSocketOption(QAbstractSocket::LowDelayOption, 1);
 
+    pipeliningSupported = QHttpNetworkConnectionChannel::PipeliningSupportUnknown;
+
     // ### FIXME: if the server closes the connection unexpectedly, we shouldn't send the same broken request again!
     //channels[i].reconnectAttempts = 2;
     if (!pendingEncrypt) {
@@ -650,7 +779,7 @@ void QHttpNetworkConnectionChannel::_q_error(QAbstractSocket::SocketError socket
         // while "Reading" the _q_disconnected() will handle this.
         if (state != QHttpNetworkConnectionChannel::IdleState && state != QHttpNetworkConnectionChannel::ReadingState) {
             if (reconnectAttempts-- > 0) {
-                connection->d_func()->resendCurrentRequest(socket);
+                closeAndResendCurrentRequest();
                 return;
             } else {
                 send2Reply = true;
@@ -663,7 +792,7 @@ void QHttpNetworkConnectionChannel::_q_error(QAbstractSocket::SocketError socket
     case QAbstractSocket::SocketTimeoutError:
         // try to reconnect/resend before sending an error.
         if (state == QHttpNetworkConnectionChannel::WritingState && (reconnectAttempts-- > 0)) {
-            connection->d_func()->resendCurrentRequest(socket);
+            closeAndResendCurrentRequest();
             return;
         }
         send2Reply = true;
@@ -701,7 +830,7 @@ void QHttpNetworkConnectionChannel::_q_error(QAbstractSocket::SocketError socket
 #ifndef QT_NO_NETWORKPROXY
 void QHttpNetworkConnectionChannel::_q_proxyAuthenticationRequired(const QNetworkProxy &proxy, QAuthenticator* auth)
 {
-    emit connection->proxyAuthenticationRequired(proxy, auth, connection);
+    connection->d_func()->emitProxyAuthenticationRequired(this, proxy, auth);
 }
 #endif
 
diff --git a/src/network/access/qhttpnetworkconnectionchannel_p.h b/src/network/access/qhttpnetworkconnectionchannel_p.h
index 498fda9..220b72c 100644
--- a/src/network/access/qhttpnetworkconnectionchannel_p.h
+++ b/src/network/access/qhttpnetworkconnectionchannel_p.h
@@ -82,10 +82,14 @@ class QHttpNetworkReply;
 class QByteArray;
 class QHttpNetworkConnection;
 
+#ifndef HttpMessagePair
+typedef QPair<QHttpNetworkRequest, QHttpNetworkReply*> HttpMessagePair;
+#endif
+
 class QHttpNetworkConnectionChannel : public QObject {
     Q_OBJECT
 public:
-        enum ChannelState {
+    enum ChannelState {
         IdleState = 0,          // ready to send request
         ConnectingState = 1,    // connecting to host
         WritingState = 2,       // writing the data
@@ -112,12 +116,24 @@ public:
     bool ignoreAllSslErrors;
     QList<QSslError> ignoreSslErrorsList;
 #endif
+
+    // HTTP pipelining -> http://en.wikipedia.org/wiki/Http_pipelining
+    enum PipeliningSupport {
+        PipeliningSupportUnknown, // default for a new connection
+        PipeliningProbablySupported, // after having received a server response that indicates support
+        PipeliningNotSupported // currently not used
+    };
+    PipeliningSupport pipeliningSupported;
+    QList<HttpMessagePair> alreadyPipelinedRequests;
+
+
     QHttpNetworkConnectionChannel() : socket(0), state(IdleState), reply(0), written(0), bytesTotal(0), resendCurrent(false),
     lastStatus(0), pendingEncrypt(false), reconnectAttempts(2),
     authMehtod(QAuthenticatorPrivate::None), proxyAuthMehtod(QAuthenticatorPrivate::None)
 #ifndef QT_NO_OPENSSL
     , ignoreAllSslErrors(false)
 #endif
+    , pipeliningSupported(PipeliningSupportUnknown)
     , connection(0)
     {}
 
@@ -126,13 +142,24 @@ public:
 
     void init();
     void close();
+
     bool sendRequest();
     void receiveReply();
+
     bool ensureConnection();
+
     bool expand(bool dataComplete);
     void allDone(); // reply header + body have been read
     void handleStatus(); // called from allDone()
 
+    void pipelineInto(HttpMessagePair &pair);
+    void requeueCurrentlyPipelinedRequests();
+    void detectPipeliningSupport();
+
+    void closeAndResendCurrentRequest();
+
+    void eatWhitespace();
+
     protected slots:
     void _q_bytesWritten(qint64 bytes); // proceed sending
     void _q_readyRead(); // pending data to read
diff --git a/src/network/access/qhttpnetworkreply.cpp b/src/network/access/qhttpnetworkreply.cpp
index cd843ab..eb8ecb5 100644
--- a/src/network/access/qhttpnetworkreply.cpp
+++ b/src/network/access/qhttpnetworkreply.cpp
@@ -418,20 +418,26 @@ qint64 QHttpNetworkReplyPrivate::readStatus(QAbstractSocket *socket)
             }
             bool ok = parseStatus(fragment);
             state = ReadingHeaderState;
-            fragment.clear(); // next fragment
-
-            if (!ok)
+            fragment.clear();
+            if (!ok) {
                 return -1;
+            }
             break;
         } else {
             c = 0;
-            bytes += socket->read(&c, 1);
+            int haveRead = socket->read(&c, 1);
+            if (haveRead == -1)
+                return -1;
+            bytes += haveRead;
             fragment.append(c);
         }
 
         // is this a valid reply?
         if (fragment.length() >= 5 && !fragment.startsWith("HTTP/"))
+        {
+            fragment.clear();
             return -1;
+        }
 
     }
 
@@ -568,7 +574,6 @@ qint64 QHttpNetworkReplyPrivate::readBodyFast(QAbstractSocket *socket, QByteData
 
     if (contentRead + haveRead == bodyLength) {
         state = AllDoneState;
-        socket->readAll(); // Read the rest to clean (CRLF) ### will break pipelining
     }
 
     contentRead += haveRead;
@@ -588,8 +593,6 @@ qint64 QHttpNetworkReplyPrivate::readBody(QAbstractSocket *socket, QByteDataBuff
     } else {
         bytes += readReplyBodyRaw(socket, out, socket->bytesAvailable());
     }
-    if (state == AllDoneState)
-        socket->readAll(); // Read the rest to clean (CRLF) ### will break pipelining
     contentRead += bytes;
     return bytes;
 }
diff --git a/src/network/access/qhttpnetworkrequest.cpp b/src/network/access/qhttpnetworkrequest.cpp
index 693e1f4..8b2c4e9 100644
--- a/src/network/access/qhttpnetworkrequest.cpp
+++ b/src/network/access/qhttpnetworkrequest.cpp
@@ -49,7 +49,7 @@ QT_BEGIN_NAMESPACE
 QHttpNetworkRequestPrivate::QHttpNetworkRequestPrivate(QHttpNetworkRequest::Operation op,
         QHttpNetworkRequest::Priority pri, const QUrl &newUrl)
     : QHttpNetworkHeaderPrivate(newUrl), operation(op), priority(pri), uploadByteDevice(0),
-      autoDecompress(false)
+      autoDecompress(false), pipeliningAllowed(false)
 {
 }
 
@@ -60,6 +60,7 @@ QHttpNetworkRequestPrivate::QHttpNetworkRequestPrivate(const QHttpNetworkRequest
     priority = other.priority;
     uploadByteDevice = other.uploadByteDevice;
     autoDecompress = other.autoDecompress;
+    pipeliningAllowed = other.pipeliningAllowed;
 }
 
 QHttpNetworkRequestPrivate::~QHttpNetworkRequestPrivate()
@@ -239,6 +240,16 @@ void QHttpNetworkRequest::setPriority(Priority priority)
     d->priority = priority;
 }
 
+bool QHttpNetworkRequest::isPipeliningAllowed() const
+{
+    return d->pipeliningAllowed;
+}
+
+void QHttpNetworkRequest::setPipeliningAllowed(bool b)
+{
+    d->pipeliningAllowed = b;
+}
+
 void QHttpNetworkRequest::setUploadByteDevice(QNonContiguousByteDevice *bd)
 {
     d->uploadByteDevice = bd;
diff --git a/src/network/access/qhttpnetworkrequest_p.h b/src/network/access/qhttpnetworkrequest_p.h
index d4c21e5..2468be9 100644
--- a/src/network/access/qhttpnetworkrequest_p.h
+++ b/src/network/access/qhttpnetworkrequest_p.h
@@ -106,6 +106,9 @@ public:
     Priority priority() const;
     void setPriority(Priority priority);
 
+    bool isPipeliningAllowed() const;
+    void setPipeliningAllowed(bool b);
+
     void setUploadByteDevice(QNonContiguousByteDevice *bd);
     QNonContiguousByteDevice* uploadByteDevice() const;
 
@@ -133,6 +136,7 @@ public:
     QHttpNetworkRequest::Priority priority;
     mutable QNonContiguousByteDevice* uploadByteDevice;
     bool autoDecompress;
+    bool pipeliningAllowed;
 };
 
 
diff --git a/src/network/access/qnetworkaccesshttpbackend.cpp b/src/network/access/qnetworkaccesshttpbackend.cpp
index 479990a..fd47b34 100644
--- a/src/network/access/qnetworkaccesshttpbackend.cpp
+++ b/src/network/access/qnetworkaccesshttpbackend.cpp
@@ -516,6 +516,9 @@ void QNetworkAccessHttpBackend::postRequest()
         return;    // no need to send the request! :)
     }
 
+    if (request().attribute(QNetworkRequest::HttpPipeliningAllowedAttribute).toBool() == true)
+        httpRequest.setPipeliningAllowed(true);
+
     httpReply = http->sendRequest(httpRequest);
     httpReply->setParent(this);
 #ifndef QT_NO_OPENSSL
diff --git a/src/network/access/qnetworkaccessmanager.cpp b/src/network/access/qnetworkaccessmanager.cpp
index ce5f6c7..839bf31 100644
--- a/src/network/access/qnetworkaccessmanager.cpp
+++ b/src/network/access/qnetworkaccessmanager.cpp
@@ -804,7 +804,13 @@ void QNetworkAccessManagerPrivate::proxyAuthenticationRequired(QNetworkAccessBac
                                                                QAuthenticator *authenticator)
 {
     Q_Q(QNetworkAccessManager);
-
+    // ### FIXME Tracking of successful authentications
+    // This code is a bit broken right now for SOCKS authentication
+    // first request: proxyAuthenticationRequired gets emitted, credentials gets saved
+    // second request: (proxy != backend->reply->lastProxyAuthentication) does not evaluate to true,
+    //      proxyAuthenticationRequired gets emitted again
+    // possible solution: some tracking inside the authenticator
+    //      or a new function proxyAuthenticationSucceeded(true|false)
     if (proxy != backend->reply->lastProxyAuthentication) {
         QNetworkAuthenticationCredential *cred = fetchCachedCredentials(proxy);
         if (cred) {
diff --git a/src/network/access/qnetworkrequest.cpp b/src/network/access/qnetworkrequest.cpp
index 721f8c4..7d838a3 100644
--- a/src/network/access/qnetworkrequest.cpp
+++ b/src/network/access/qnetworkrequest.cpp
@@ -167,6 +167,11 @@ QT_BEGIN_NAMESPACE
         When using this flag with sequential upload data, the ContentLengthHeader
         header must be set.
 
+     \value HttpPipeliningAllowed
+        Requests only, type: QVariant::Bool (default: false)
+        Indicates whether the QNetworkAccessManager code is
+        allowed to use HTTP pipelining with this request.
+
     \value User
         Special type. Additional information can be passed in
         QVariants with types ranging from User to UserMax. The default
diff --git a/src/network/access/qnetworkrequest.h b/src/network/access/qnetworkrequest.h
index aaeed48..fb5ea52 100644
--- a/src/network/access/qnetworkrequest.h
+++ b/src/network/access/qnetworkrequest.h
@@ -76,6 +76,7 @@ public:
         CacheSaveControlAttribute,
         SourceIsFromCacheAttribute,
         DoNotBufferUploadDataAttribute,
+        HttpPipeliningAllowedAttribute,
 
         User = 1000,
         UserMax = 32767
diff --git a/tests/auto/qhttpnetworkconnection/tst_qhttpnetworkconnection.cpp b/tests/auto/qhttpnetworkconnection/tst_qhttpnetworkconnection.cpp
index 4f24721..c4c33d5 100644
--- a/tests/auto/qhttpnetworkconnection/tst_qhttpnetworkconnection.cpp
+++ b/tests/auto/qhttpnetworkconnection/tst_qhttpnetworkconnection.cpp
@@ -99,6 +99,9 @@ private Q_SLOTS:
     void get401_data();
     void get401();
 
+    void getMultiple_data();
+    void getMultiple();
+    void getMultipleWithPipeliningAndMultiplePriorities();
 };
 
 
@@ -763,5 +766,123 @@ void tst_QHttpNetworkConnection::nossl()
 }
 #endif
 
+
+void tst_QHttpNetworkConnection::getMultiple_data()
+{
+    QTest::addColumn<quint16>("connectionCount");
+    QTest::addColumn<bool>("pipeliningAllowed");
+    // send 100 requests. apache will usually force-close after 100 requests in a single tcp connection
+    QTest::addColumn<int>("requestCount");
+
+    QTest::newRow("6 connections, no pipelining, 100 requests")  << quint16(6) << false << 100;
+    QTest::newRow("1 connection, no pipelining, 100 requests")  << quint16(1) << false << 100;
+    QTest::newRow("6 connections, pipelining allowed, 100 requests")  << quint16(2) << true << 100;
+    QTest::newRow("1 connection, pipelining allowed, 100 requests")  << quint16(1) << true << 100;
+}
+
+void tst_QHttpNetworkConnection::getMultiple()
+{
+    QFETCH(quint16, connectionCount);
+    QFETCH(bool, pipeliningAllowed);
+    QFETCH(int, requestCount);
+
+    QHttpNetworkConnection connection(connectionCount, QtNetworkSettings::serverName());
+
+    QList<QHttpNetworkRequest*> requests;
+    QList<QHttpNetworkReply*> replies;
+
+    for (int i = 0; i < requestCount; i++) {
+        // depending on what you use the results will vary.
+        // for the "real" results, use a URL that has "internet latency" for you. Then (6 connections, pipelining) will win.
+        // for LAN latency, you will possibly get that (1 connection, no pipelining) is the fastest
+        QHttpNetworkRequest *request = new QHttpNetworkRequest("http://" + QtNetworkSettings::serverName() + "/qtest/rfc3252.txt");
+        // located in Berlin:
+        //QHttpNetworkRequest *request = new QHttpNetworkRequest(QUrl("http://klinsmann.nokia.trolltech.de/~berlin/qtcreatorad.gif"));
+        if (pipeliningAllowed)
+            request->setPipeliningAllowed(true);
+        requests.append(request);
+        QHttpNetworkReply *reply = connection.sendRequest(*request);
+        replies.append(reply);
+    }
+
+    QTime stopWatch;
+    stopWatch.start();
+    int finishedCount = 0;
+    do {
+        QCoreApplication::instance()->processEvents();
+        if (stopWatch.elapsed() >= 60000)
+            break;
+
+        finishedCount = 0;
+        for (int i = 0; i < replies.length(); i++)
+            if (replies.at(i)->isFinished())
+                finishedCount++;
+
+    } while (finishedCount != replies.length());
+
+    // redundant
+    for (int i = 0; i < replies.length(); i++)
+        QVERIFY(replies.at(i)->isFinished());
+
+    qDebug() << "===" << stopWatch.elapsed() << "msec ===";
+
+    qDeleteAll(requests);
+    qDeleteAll(replies);
+}
+
+void tst_QHttpNetworkConnection::getMultipleWithPipeliningAndMultiplePriorities()
+{
+    quint16 requestCount = 100;
+
+    // use 2 connections.
+    QHttpNetworkConnection connection(2, QtNetworkSettings::serverName());
+
+    QList<QHttpNetworkRequest*> requests;
+    QList<QHttpNetworkReply*> replies;
+
+    for (int i = 0; i < requestCount; i++) {
+
+        QHttpNetworkRequest *request = new QHttpNetworkRequest("http://" + QtNetworkSettings::serverName() + "/qtest/rfc3252.txt");
+        
+        if (i % 2 || i % 3)
+            request->setPipeliningAllowed(true);
+        
+        if (i % 3)
+            request->setPriority(QHttpNetworkRequest::HighPriority);
+        else if (i % 5)
+            request->setPriority(QHttpNetworkRequest::NormalPriority);
+        else if (i % 7)
+            request->setPriority(QHttpNetworkRequest::LowPriority);
+        
+        requests.append(request);
+        QHttpNetworkReply *reply = connection.sendRequest(*request);
+        replies.append(reply);
+    }
+
+    QTime stopWatch;
+    stopWatch.start();
+    int finishedCount = 0;
+    do {
+        QCoreApplication::instance()->processEvents();
+        if (stopWatch.elapsed() >= 60000)
+            break;
+
+        finishedCount = 0;
+        for (int i = 0; i < replies.length(); i++)
+            if (replies.at(i)->isFinished())
+                finishedCount++;
+
+    } while (finishedCount != replies.length());
+
+    // redundant
+    for (int i = 0; i < replies.length(); i++)
+        QVERIFY(replies.at(i)->isFinished());
+
+    qDebug() << "===" << stopWatch.elapsed() << "msec ===";
+
+    qDeleteAll(requests);
+    qDeleteAll(replies);
+}
+
 QTEST_MAIN(tst_QHttpNetworkConnection)
 #include "tst_qhttpnetworkconnection.moc"
diff --git a/tests/auto/qnetworkreply/tst_qnetworkreply.cpp b/tests/auto/qnetworkreply/tst_qnetworkreply.cpp
index d339803..7a9d016 100644
--- a/tests/auto/qnetworkreply/tst_qnetworkreply.cpp
+++ b/tests/auto/qnetworkreply/tst_qnetworkreply.cpp
@@ -2254,8 +2254,11 @@ void tst_QNetworkReply::ioGetFromHttpBrokenServer_data()
     QTest::addColumn<bool>("doDisconnect");
 
     QTest::newRow("no-newline") << QByteArray("Hello World") << false;
-    QTest::newRow("just-newline") << QByteArray("\r\n") << false;
-    QTest::newRow("just-2newline") << QByteArray("\r\n\r\n") << false;
+
+    // these are OK now, we just eat the lonely newlines
+    //QTest::newRow("just-newline") << QByteArray("\r\n") << false;
+    //QTest::newRow("just-2newline") << QByteArray("\r\n\r\n") << false;
+
     QTest::newRow("with-newlines") << QByteArray("Long first line\r\nLong second line") << false;
     QTest::newRow("with-newlines2") << QByteArray("\r\nSecond line") << false;
     QTest::newRow("with-newlines3") << QByteArray("ICY\r\nSecond line") << false;
@@ -2931,7 +2934,7 @@ void tst_QNetworkReply::ioPostToHttpFromSocket()
     QSignalSpy authenticationRequiredSpy(&manager, SIGNAL(authenticationRequired(QNetworkReply*,QAuthenticator*)));
     QSignalSpy proxyAuthenticationRequiredSpy(&manager, SIGNAL(proxyAuthenticationRequired(QNetworkProxy,QAuthenticator*)));
 
-    QTestEventLoop::instance().enterLoop(1);
+    QTestEventLoop::instance().enterLoop(3);
 
     disconnect(&manager, SIGNAL(proxyAuthenticationRequired(QNetworkProxy,QAuthenticator*)),
                this, SLOT(proxyAuthenticationRequired(QNetworkProxy,QAuthenticator*)));