QNAM HTTP Pipelining

HTTP Pipelining should improve the performance of HTTP requests
for high latency network links. Since some servers/proxies could
have problems with it, it is disabled by default.
Set the HttpPipeliningAllowed attribute of a QNetworkRequest
to enable it for that request.

Reviewed-by: Thiago

1 files changed, 153 insertions, 24 deletions

diff --git a/src/network/access/qhttpnetworkconnectionchannel.cpp b/src/network/access/qhttpnetworkconnectionchannel.cpp
index 2d33527..221c27c 100644
--- a/src/network/access/qhttpnetworkconnectionchannel.cpp
+++ b/src/network/access/qhttpnetworkconnectionchannel.cpp
@@ -165,8 +165,8 @@ bool QHttpNetworkConnectionChannel::sendRequest()
 
             bytesTotal = request.contentLength();
         } else {
-            socket->flush(); // ### Remove this when pipelining is implemented. We want less TCP packets!
             state = QHttpNetworkConnectionChannel::WaitingState;
+            sendRequest();
             break;
         }
         // write the initial chunk together with the headers
@@ -242,6 +242,11 @@ bool QHttpNetworkConnectionChannel::sendRequest()
         if (uploadByteDevice) {
             QObject::disconnect(uploadByteDevice, SIGNAL(readyRead()), this, SLOT(_q_uploadDataReadyRead()));
         }
+
+        // HTTP pipelining
+        connection->d_func()->fillPipeline(socket);
+        socket->flush();
+
         // ensure we try to receive a reply in all cases, even if _q_readyRead_ hat not been called
         // this is needed if the sends an reply before we have finished sending the request. In that
         // case receiveReply had been called before but ignored the server reply
@@ -276,7 +281,7 @@ void QHttpNetworkConnectionChannel::receiveReply()
             } else {
                 // try to reconnect/resend before sending an error.
                 if (reconnectAttempts-- > 0) {
-                    connection->d_func()->resendCurrentRequest(socket);
+                    closeAndResendCurrentRequest();
                 } else if (reply) {
                     reply->d_func()->errorString = connection->d_func()->errorDetail(QNetworkReply::RemoteHostClosedError, socket);
                     emit reply->finishedWithError(QNetworkReply::RemoteHostClosedError, reply->d_func()->errorString);
@@ -292,14 +297,20 @@ void QHttpNetworkConnectionChannel::receiveReply()
         switch (state) {
         case QHttpNetworkReplyPrivate::NothingDoneState:
         case QHttpNetworkReplyPrivate::ReadingStatusState: {
+            eatWhitespace();
             qint64 statusBytes = reply->d_func()->readStatus(socket);
-            if (statusBytes == -1) {
-                // error reading the status, close the socket and emit error
-                socket->close();
+            if (statusBytes == -1 && reconnectAttempts <= 0) {
+                // too many errors reading/receiving/parsing the status, close the socket and emit error
+                close();
                 reply->d_func()->errorString = connection->d_func()->errorDetail(QNetworkReply::ProtocolFailure, socket);
                 emit reply->finishedWithError(QNetworkReply::ProtocolFailure, reply->d_func()->errorString);
                 QMetaObject::invokeMethod(connection, "_q_startNextRequest", Qt::QueuedConnection);
                 break;
+            } else if (statusBytes == -1) {
+                reconnectAttempts--;
+                reply->d_func()->clear();
+                closeAndResendCurrentRequest();
+                break;
             }
             bytes += statusBytes;
             lastStatus = reply->d_func()->statusCode;
@@ -410,6 +421,9 @@ bool QHttpNetworkConnectionChannel::ensureConnection()
         state = QHttpNetworkConnectionChannel::ConnectingState;
         pendingEncrypt = connection->d_func()->encrypt;
 
+        // reset state
+        pipeliningSupported = PipeliningSupportUnknown;
+
         // This workaround is needed since we use QAuthenticator for NTLM authentication. The "phase == Done"
         // is the usual criteria for emitting authentication signals. The "phase" is set to "Done" when the
         // last header for Authorization is generated by the QAuthenticator. Basic & Digest logic does not
@@ -520,6 +534,92 @@ void QHttpNetworkConnectionChannel::allDone()
     // reset the reconnection attempts after we receive a complete reply.
     // in case of failures, each channel will attempt two reconnects before emitting error.
     reconnectAttempts = 2;
+
+    detectPipeliningSupport();
+
+    // move next from pipeline to current request
+    if (!alreadyPipelinedRequests.isEmpty()) {
+        if (resendCurrent || reply->d_func()->connectionCloseEnabled() || socket->state() != QAbstractSocket::Connected) {
+            // move the pipelined ones back to the main queue
+            requeueCurrentlyPipelinedRequests();
+        } else {
+            // there were requests pipelined in and we can continue
+            HttpMessagePair messagePair = alreadyPipelinedRequests.takeFirst();
+
+            request = messagePair.first;
+            reply = messagePair.second;
+            state = QHttpNetworkConnectionChannel::ReadingState;
+            resendCurrent = false;
+
+            written = 0; // message body, excluding the header, irrelevant here
+            bytesTotal = 0; // message body total, excluding the header, irrelevant here
+
+            // pipeline even more
+            connection->d_func()->fillPipeline(socket);
+
+            // continue reading
+            receiveReply();
+        }
+    } else if (alreadyPipelinedRequests.isEmpty() && socket->bytesAvailable() > 0) {
+        eatWhitespace();
+        // this is weird. we had nothing pipelined but still bytes available. better close it.
+        if (socket->bytesAvailable() > 0)
+            close();
+        QMetaObject::invokeMethod(connection, "_q_startNextRequest", Qt::QueuedConnection);
+    } else if (alreadyPipelinedRequests.isEmpty()) {
+        QMetaObject::invokeMethod(connection, "_q_startNextRequest", Qt::QueuedConnection);
+    }
+}
+
+void QHttpNetworkConnectionChannel::detectPipeliningSupport()
+{
+    // detect HTTP Pipelining support
+    QByteArray serverHeaderField = reply->headerField("Server");
+    if (
+            // check for broken servers in server reply header
+            // this is adapted from http://mxr.mozilla.org/firefox/ident?i=SupportsPipelining
+            (!serverHeaderField.contains("Microsoft-IIS/4."))
+            && (!serverHeaderField.contains("Microsoft-IIS/5."))
+            && (!serverHeaderField.contains("Netscape-Enterprise/3."))
+            // check for HTTP/1.1
+            && (reply->d_func()->majorVersion == 1 && reply->d_func()->minorVersion == 1)
+            // check for not having connection close
+            && (!reply->d_func()->connectionCloseEnabled())
+            // check if it is still connected
+            && (socket->state() == QAbstractSocket::Connected)
+            ) {
+        pipeliningSupported = QHttpNetworkConnectionChannel::PipeliningProbablySupported;
+    } else {
+        pipeliningSupported = QHttpNetworkConnectionChannel::PipeliningSupportUnknown;
+    }
+}
+
+// called when the connection broke and we need to queue some pipelined requests again
+void QHttpNetworkConnectionChannel::requeueCurrentlyPipelinedRequests()
+{
+    for (int i = 0; i < alreadyPipelinedRequests.length(); i++)
+        connection->d_func()->requeueRequest(alreadyPipelinedRequests.at(i));
+    alreadyPipelinedRequests.clear();
+
+    close();
+    QMetaObject::invokeMethod(connection, "_q_startNextRequest", Qt::QueuedConnection);
+}
+
+void QHttpNetworkConnectionChannel::eatWhitespace()
+{
+    char c;
+    while (socket->bytesAvailable()) {
+        if (socket->peek(&c, 1) != 1)
+            return;
+
+        // read all whitespace and line endings
+        if (c == 11 || c == '\n' || c == '\r' || c == ' ' || c == 31) {
+            socket->read(&c, 1);
+            continue;
+        } else {
+            break;
+        }
+    }
 }
 
 void QHttpNetworkConnectionChannel::handleStatus()
@@ -531,8 +631,8 @@ void QHttpNetworkConnectionChannel::handleStatus()
     bool resend = false;
 
     switch (statusCode) {
-    case 401:
-    case 407:
+    case 401: // auth required
+    case 407: // proxy auth required
         if (connection->d_func()->handleAuthenticateChallenge(socket, reply, (statusCode == 407), resend)) {
             if (resend) {
                 QNonContiguousByteDevice* uploadByteDevice = request.uploadByteDevice();
@@ -547,10 +647,15 @@ void QHttpNetworkConnectionChannel::handleStatus()
 
                 reply->d_func()->eraseData();
 
-                // also use async _q_startNextRequest so we dont break with closed
-                // proxy or server connections..
-                resendCurrent = true;
-                QMetaObject::invokeMethod(connection, "_q_startNextRequest", Qt::QueuedConnection);
+                if (alreadyPipelinedRequests.isEmpty()) {
+                    // this does a re-send without closing the connection
+                    resendCurrent = true;
+                    QMetaObject::invokeMethod(connection, "_q_startNextRequest", Qt::QueuedConnection);
+                } else {
+                    // we had requests pipelined.. better close the connection in closeAndResendCurrentRequest
+                    closeAndResendCurrentRequest();
+                    QMetaObject::invokeMethod(connection, "_q_startNextRequest", Qt::QueuedConnection);
+                }
             }
         } else {
             emit reply->headerChanged();
@@ -568,26 +673,51 @@ void QHttpNetworkConnectionChannel::handleStatus()
     }
 }
 
+void  QHttpNetworkConnectionChannel::pipelineInto(HttpMessagePair &pair)
+{
+    // this is only called for simple GET
+
+    QHttpNetworkRequest &request = pair.first;
+    QHttpNetworkReply *reply = pair.second;
+    if (reply) {
+        reply->d_func()->clear();
+        reply->d_func()->connection = connection;
+        reply->d_func()->autoDecompress = request.d->autoDecompress;
+    }
 
+#ifndef QT_NO_NETWORKPROXY
+    QByteArray header = QHttpNetworkRequestPrivate::header(request,
+                                                           (connection->d_func()->networkProxy.type() != QNetworkProxy::NoProxy));
+#else
+    QByteArray header = QHttpNetworkRequestPrivate::header(channels[i].request,
+                                                           false);
+#endif
+    socket->write(header);
+
+    alreadyPipelinedRequests.append(pair);
+}
+
+void QHttpNetworkConnectionChannel::closeAndResendCurrentRequest()
+{
+    requeueCurrentlyPipelinedRequests();
+    close();
+    resendCurrent = true;
+    QMetaObject::invokeMethod(connection, "_q_startNextRequest", Qt::QueuedConnection);
+}
 
 //private slots
 void QHttpNetworkConnectionChannel::_q_readyRead()
 {
-    if (!socket)
-        return; // ### error
     if (connection->d_func()->isSocketWaiting(socket) || connection->d_func()->isSocketReading(socket)) {
         state = QHttpNetworkConnectionChannel::ReadingState;
         if (reply)
             receiveReply();
     }
-    // ### error
 }
 
 void QHttpNetworkConnectionChannel::_q_bytesWritten(qint64 bytes)
 {
     Q_UNUSED(bytes);
-    if (!socket)
-        return; // ### error
     // bytes have been written to the socket. write even more of them :)
     if (connection->d_func()->isSocketWriting(socket))
         sendRequest();
@@ -596,8 +726,6 @@ void QHttpNetworkConnectionChannel::_q_bytesWritten(qint64 bytes)
 
 void QHttpNetworkConnectionChannel::_q_disconnected()
 {
-    if (!socket)
-        return; // ### error
     // read the available data before closing
     if (connection->d_func()->isSocketWaiting(socket) || connection->d_func()->isSocketReading(socket)) {
         state = QHttpNetworkConnectionChannel::ReadingState;
@@ -608,17 +736,18 @@ void QHttpNetworkConnectionChannel::_q_disconnected()
         QMetaObject::invokeMethod(connection, "_q_startNextRequest", Qt::QueuedConnection);
     }
     state = QHttpNetworkConnectionChannel::IdleState;
+
+    requeueCurrentlyPipelinedRequests();
 }
 
 
 void QHttpNetworkConnectionChannel::_q_connected()
 {
-    if (!socket)
-        return; // ### error
-
     // improve performance since we get the request sent by the kernel ASAP
     socket->setSocketOption(QAbstractSocket::LowDelayOption, 1);
 
+    pipeliningSupported = QHttpNetworkConnectionChannel::PipeliningSupportUnknown;
+
     // ### FIXME: if the server closes the connection unexpectedly, we shouldn't send the same broken request again!
     //channels[i].reconnectAttempts = 2;
     if (!pendingEncrypt) {
@@ -650,7 +779,7 @@ void QHttpNetworkConnectionChannel::_q_error(QAbstractSocket::SocketError socket
         // while "Reading" the _q_disconnected() will handle this.
         if (state != QHttpNetworkConnectionChannel::IdleState && state != QHttpNetworkConnectionChannel::ReadingState) {
             if (reconnectAttempts-- > 0) {
-                connection->d_func()->resendCurrentRequest(socket);
+                closeAndResendCurrentRequest();
                 return;
             } else {
                 send2Reply = true;
@@ -663,7 +792,7 @@ void QHttpNetworkConnectionChannel::_q_error(QAbstractSocket::SocketError socket
     case QAbstractSocket::SocketTimeoutError:
         // try to reconnect/resend before sending an error.
         if (state == QHttpNetworkConnectionChannel::WritingState && (reconnectAttempts-- > 0)) {
-            connection->d_func()->resendCurrentRequest(socket);
+            closeAndResendCurrentRequest();
             return;
         }
         send2Reply = true;
@@ -701,7 +830,7 @@ void QHttpNetworkConnectionChannel::_q_error(QAbstractSocket::SocketError socket
 #ifndef QT_NO_NETWORKPROXY
 void QHttpNetworkConnectionChannel::_q_proxyAuthenticationRequired(const QNetworkProxy &proxy, QAuthenticator* auth)
 {
-    emit connection->proxyAuthenticationRequired(proxy, auth, connection);
+    connection->d_func()->emitProxyAuthenticationRequired(this, proxy, auth);
 }
 #endif