summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorMarkus Goetz <Markus.Goetz@nokia.com>2009-08-17 14:15:18 (GMT)
committerMarkus Goetz <Markus.Goetz@nokia.com>2009-08-20 08:52:37 (GMT)
commit67376be28ca51930ff0f4fad2dd58f53968655a9 (patch)
tree06da6fb0a834d3cbd70147098476f32d06cbd48b
parent09138f1f4eb45c196506ab94a455dc47cec3575a (diff)
downloadQt-67376be28ca51930ff0f4fad2dd58f53968655a9.zip
Qt-67376be28ca51930ff0f4fad2dd58f53968655a9.tar.gz
Qt-67376be28ca51930ff0f4fad2dd58f53968655a9.tar.bz2
QNAM HTTP Pipelining
HTTP Pipelining should improve the performance of HTTP requests for high latency network links. Since some servers/proxies could have problems with it, it is disabled by default. Set the HttpPipeliningAllowed attribute of a QNetworkRequest to enable it for that request. Reviewed-by: Thiago
-rw-r--r--demos/browser/networkaccessmanager.h9
-rw-r--r--src/network/access/qhttpnetworkconnection.cpp165
-rw-r--r--src/network/access/qhttpnetworkconnection_p.h10
-rw-r--r--src/network/access/qhttpnetworkconnectionchannel.cpp177
-rw-r--r--src/network/access/qhttpnetworkconnectionchannel_p.h29
-rw-r--r--src/network/access/qhttpnetworkreply.cpp17
-rw-r--r--src/network/access/qhttpnetworkrequest.cpp13
-rw-r--r--src/network/access/qhttpnetworkrequest_p.h4
-rw-r--r--src/network/access/qnetworkaccesshttpbackend.cpp3
-rw-r--r--src/network/access/qnetworkaccessmanager.cpp8
-rw-r--r--src/network/access/qnetworkrequest.cpp5
-rw-r--r--src/network/access/qnetworkrequest.h1
-rw-r--r--tests/auto/qhttpnetworkconnection/tst_qhttpnetworkconnection.cpp121
-rw-r--r--tests/auto/qnetworkreply/tst_qnetworkreply.cpp9
14 files changed, 519 insertions, 52 deletions
diff --git a/demos/browser/networkaccessmanager.h b/demos/browser/networkaccessmanager.h
index 381cb50..4c4603e 100644
--- a/demos/browser/networkaccessmanager.h
+++ b/demos/browser/networkaccessmanager.h
@@ -43,6 +43,7 @@
#define NETWORKACCESSMANAGER_H
#include <QtNetwork/QNetworkAccessManager>
+#include <QtNetwork/QNetworkRequest>
class NetworkAccessManager : public QNetworkAccessManager
{
@@ -51,6 +52,14 @@ class NetworkAccessManager : public QNetworkAccessManager
public:
NetworkAccessManager(QObject *parent = 0);
+ // this is a temporary hack until we properly use the pipelining flags from QtWebkit
+ // pipeline everything! :)
+ virtual QNetworkReply* createRequest ( Operation op, const QNetworkRequest & req, QIODevice * outgoingData = 0 ) {
+ QNetworkRequest request = req; // copy so we can modify
+ request.setAttribute(QNetworkRequest::HttpPipeliningAllowedAttribute, true);
+ return QNetworkAccessManager::createRequest(op, request, outgoingData);
+ }
+
private:
QList<QString> sslTrustedHostList;
diff --git a/src/network/access/qhttpnetworkconnection.cpp b/src/network/access/qhttpnetworkconnection.cpp
index 6888891..397e0b5 100644
--- a/src/network/access/qhttpnetworkconnection.cpp
+++ b/src/network/access/qhttpnetworkconnection.cpp
@@ -67,6 +67,11 @@ QT_BEGIN_NAMESPACE
const int QHttpNetworkConnectionPrivate::defaultChannelCount = 6;
+// the maximum amount of requests that might be pipelined into a socket
+// from what was suggested, 3 seems to be OK
+const int QHttpNetworkConnectionPrivate::defaultPipelineLength = 3;
+
+
QHttpNetworkConnectionPrivate::QHttpNetworkConnectionPrivate(const QString &hostName, quint16 port, bool encrypt)
: hostName(hostName), port(port), encrypt(encrypt),
channelCount(defaultChannelCount),
@@ -414,7 +419,24 @@ QHttpNetworkReply* QHttpNetworkConnectionPrivate::queueRequest(const QHttpNetwor
return reply;
}
-void QHttpNetworkConnectionPrivate::unqueueAndSendRequest(QAbstractSocket *socket)
+void QHttpNetworkConnectionPrivate::requeueRequest(const HttpMessagePair &pair)
+{
+ Q_Q(QHttpNetworkConnection);
+
+ QHttpNetworkRequest request = pair.first;
+ switch (request.priority()) {
+ case QHttpNetworkRequest::HighPriority:
+ highPriorityQueue.prepend(pair);
+ break;
+ case QHttpNetworkRequest::NormalPriority:
+ case QHttpNetworkRequest::LowPriority:
+ lowPriorityQueue.prepend(pair);
+ break;
+ }
+ QMetaObject::invokeMethod(q, "_q_startNextRequest", Qt::QueuedConnection);
+}
+
+void QHttpNetworkConnectionPrivate::dequeueAndSendRequest(QAbstractSocket *socket)
{
Q_ASSERT(socket);
@@ -428,8 +450,9 @@ void QHttpNetworkConnectionPrivate::unqueueAndSendRequest(QAbstractSocket *socke
channels[i].request = messagePair.first;
channels[i].reply = messagePair.second;
- channels[i].sendRequest();
+ // remove before sendRequest! else we might pipeline the same request again
highPriorityQueue.removeAt(j);
+ channels[i].sendRequest();
return;
}
}
@@ -441,23 +464,113 @@ void QHttpNetworkConnectionPrivate::unqueueAndSendRequest(QAbstractSocket *socke
prepareRequest(messagePair);
channels[i].request = messagePair.first;
channels[i].reply = messagePair.second;
- channels[i].sendRequest();
+ // remove before sendRequest! else we might pipeline the same request again
lowPriorityQueue.removeAt(j);
+ channels[i].sendRequest();
return;
}
}
}
-void QHttpNetworkConnectionPrivate::resendCurrentRequest(QAbstractSocket *socket)
+// this is called from _q_startNextRequest and when a request has been sent down a socket from the channel
+void QHttpNetworkConnectionPrivate::fillPipeline(QAbstractSocket *socket)
{
- Q_Q(QHttpNetworkConnection);
- Q_ASSERT(socket);
+ // return fast if there is nothing to pipeline
+ if (highPriorityQueue.isEmpty() && lowPriorityQueue.isEmpty())
+ return;
+
int i = indexOf(socket);
- channels[i].close();
- channels[i].resendCurrent = true;
- QMetaObject::invokeMethod(q, "_q_startNextRequest", Qt::QueuedConnection);
+
+ bool highPriorityQueueProcessingDone = false;
+ bool lowPriorityQueueProcessingDone = false;
+
+ while (!highPriorityQueueProcessingDone && !lowPriorityQueueProcessingDone) {
+ // this loop runs once per request we intend to pipeline in.
+
+ if (channels[i].pipeliningSupported != QHttpNetworkConnectionChannel::PipeliningProbablySupported)
+ return;
+
+ // the current request that is in must already support pipelining
+ if (!channels[i].request.isPipeliningAllowed())
+ return;
+
+ // the current request must be a idempotent (right now we only check GET)
+ if (channels[i].request.operation() != QHttpNetworkRequest::Get)
+ return;
+
+ // check if socket is connected
+ if (socket->state() != QAbstractSocket::ConnectedState)
+ return;
+
+ // check for resendCurrent
+ if (channels[i].resendCurrent)
+ return;
+
+ // we do not like authentication stuff
+ // ### make sure to be OK with this in later releases
+ if (!channels[i].authenticator.isNull() || !channels[i].authenticator.user().isEmpty())
+ return;
+ if (!channels[i].proxyAuthenticator.isNull() || !channels[i].proxyAuthenticator.user().isEmpty())
+ return;
+
+ // check for pipeline length
+ if (channels[i].alreadyPipelinedRequests.length() >= defaultPipelineLength)
+ return;
+
+ // must be in ReadingState or WaitingState
+ if (! (channels[i].state == QHttpNetworkConnectionChannel::WaitingState
+ || channels[i].state == QHttpNetworkConnectionChannel::ReadingState))
+ return;
+
+ highPriorityQueueProcessingDone = fillPipeline(highPriorityQueue, channels[i]);
+ // not finished with highPriorityQueue? then loop again
+ if (!highPriorityQueueProcessingDone)
+ continue;
+ // highPriorityQueue was processed, now deal with the lowPriorityQueue
+ lowPriorityQueueProcessingDone = fillPipeline(lowPriorityQueue, channels[i]);
+ }
+}
+
+// returns true when the processing of a queue has been done
+bool QHttpNetworkConnectionPrivate::fillPipeline(QList<HttpMessagePair> &queue, QHttpNetworkConnectionChannel &channel)
+{
+ if (queue.isEmpty())
+ return true;
+
+ for (int i = 0; i < queue.length(); i++) {
+ HttpMessagePair messagePair = queue.at(i);
+ const QHttpNetworkRequest &request = messagePair.first;
+
+ // we currently do not support pipelining if HTTP authentication is used
+ if (!request.url().userInfo().isEmpty())
+ continue;
+
+ // take only GET requests
+ if (request.operation() != QHttpNetworkRequest::Get)
+ continue;
+
+ if (!request.isPipeliningAllowed())
+ continue;
+
+ // remove it from the queue
+ queue.takeAt(i);
+ // we modify the queue we iterate over here, but since we return from the function
+ // afterwards this is fine.
+
+ // actually send it
+ if (!messagePair.second->d_func()->requestIsPrepared)
+ prepareRequest(messagePair);
+ channel.pipelineInto(messagePair);
+
+ // return false because we processed something and need to process again
+ return false;
+ }
+
+ // return true, the queue has been processed and not changed
+ return true;
}
+
QString QHttpNetworkConnectionPrivate::errorDetail(QNetworkReply::NetworkError errorCode, QAbstractSocket* socket)
{
Q_ASSERT(socket);
@@ -560,16 +673,29 @@ void QHttpNetworkConnectionPrivate::_q_startNextRequest()
break;
}
}
- if (!socket)
- return; // this will be called after finishing current request.
- unqueueAndSendRequest(socket);
+
+ // this socket is free,
+ if (socket)
+ dequeueAndSendRequest(socket);
+
+ // try to push more into all sockets
+ // ### FIXME we should move this to the beginning of the function
+ // as soon as QtWebkit is properly using the pipelining
+ // (e.g. not for XMLHttpRequest or the first page load)
+ // ### FIXME we should also divide the requests more even
+ // on the connected sockets
+ //tryToFillPipeline(socket);
+ // return fast if there is nothing to pipeline
+ if (highPriorityQueue.isEmpty() && lowPriorityQueue.isEmpty())
+ return;
+ for (int j = 0; j < channelCount; j++)
+ fillPipeline(channels[j].socket);
}
void QHttpNetworkConnectionPrivate::_q_restartAuthPendingRequests()
{
// send the request using the idle socket
for (int i = 0 ; i < channelCount; ++i) {
- QAbstractSocket *socket = channels[i].socket;
if (channels[i].state == QHttpNetworkConnectionChannel::Wait4AuthState) {
channels[i].state = QHttpNetworkConnectionChannel::IdleState;
if (channels[i].reply)
@@ -739,6 +865,19 @@ void QHttpNetworkConnection::ignoreSslErrors(const QList<QSslError> &errors, int
#endif //QT_NO_OPENSSL
+#ifndef QT_NO_NETWORKPROXY
+// only called from QHttpNetworkConnectionChannel::_q_proxyAuthenticationRequired, not
+// from QHttpNetworkConnectionChannel::handleAuthenticationChallenge
+// e.g. it is for SOCKS proxies which require authentication.
+void QHttpNetworkConnectionPrivate::emitProxyAuthenticationRequired(const QHttpNetworkConnectionChannel *chan, const QNetworkProxy &proxy, QAuthenticator* auth)
+{
+ Q_Q(QHttpNetworkConnection);
+ emit q->proxyAuthenticationRequired(proxy, auth, q);
+ int i = indexOf(chan->socket);
+ copyCredentials(i, auth, true);
+}
+#endif
+
QT_END_NAMESPACE
diff --git a/src/network/access/qhttpnetworkconnection_p.h b/src/network/access/qhttpnetworkconnection_p.h
index 3f928ec..af764ed 100644
--- a/src/network/access/qhttpnetworkconnection_p.h
+++ b/src/network/access/qhttpnetworkconnection_p.h
@@ -155,6 +155,8 @@ class QHttpNetworkConnectionPrivate : public QObjectPrivate
Q_DECLARE_PUBLIC(QHttpNetworkConnection)
public:
static const int defaultChannelCount;
+ static const int defaultPipelineLength;
+
QHttpNetworkConnectionPrivate(const QString &hostName, quint16 port, bool encrypt);
QHttpNetworkConnectionPrivate(quint16 channelCount, const QString &hostName, quint16 port, bool encrypt);
~QHttpNetworkConnectionPrivate();
@@ -169,9 +171,12 @@ public:
bool isSocketReading(QAbstractSocket *socket) const;
QHttpNetworkReply *queueRequest(const QHttpNetworkRequest &request);
- void unqueueAndSendRequest(QAbstractSocket *socket);
+ void requeueRequest(const HttpMessagePair &pair); // e.g. after pipeline broke
+ void dequeueAndSendRequest(QAbstractSocket *socket);
void prepareRequest(HttpMessagePair &request);
- void resendCurrentRequest(QAbstractSocket *socket);
+
+ void fillPipeline(QAbstractSocket *socket);
+ bool fillPipeline(QList<HttpMessagePair> &queue, QHttpNetworkConnectionChannel &channel);
void copyCredentials(int fromChannel, QAuthenticator *auth, bool isProxy);
@@ -212,6 +217,7 @@ public:
#ifndef QT_NO_NETWORKPROXY
QNetworkProxy networkProxy;
+ void emitProxyAuthenticationRequired(const QHttpNetworkConnectionChannel *chan, const QNetworkProxy &proxy, QAuthenticator* auth);
#endif
//The request queues
diff --git a/src/network/access/qhttpnetworkconnectionchannel.cpp b/src/network/access/qhttpnetworkconnectionchannel.cpp
index 2d33527..221c27c 100644
--- a/src/network/access/qhttpnetworkconnectionchannel.cpp
+++ b/src/network/access/qhttpnetworkconnectionchannel.cpp
@@ -165,8 +165,8 @@ bool QHttpNetworkConnectionChannel::sendRequest()
bytesTotal = request.contentLength();
} else {
- socket->flush(); // ### Remove this when pipelining is implemented. We want less TCP packets!
state = QHttpNetworkConnectionChannel::WaitingState;
+ sendRequest();
break;
}
// write the initial chunk together with the headers
@@ -242,6 +242,11 @@ bool QHttpNetworkConnectionChannel::sendRequest()
if (uploadByteDevice) {
QObject::disconnect(uploadByteDevice, SIGNAL(readyRead()), this, SLOT(_q_uploadDataReadyRead()));
}
+
+ // HTTP pipelining
+ connection->d_func()->fillPipeline(socket);
+ socket->flush();
+
// ensure we try to receive a reply in all cases, even if _q_readyRead_ hat not been called
// this is needed if the sends an reply before we have finished sending the request. In that
// case receiveReply had been called before but ignored the server reply
@@ -276,7 +281,7 @@ void QHttpNetworkConnectionChannel::receiveReply()
} else {
// try to reconnect/resend before sending an error.
if (reconnectAttempts-- > 0) {
- connection->d_func()->resendCurrentRequest(socket);
+ closeAndResendCurrentRequest();
} else if (reply) {
reply->d_func()->errorString = connection->d_func()->errorDetail(QNetworkReply::RemoteHostClosedError, socket);
emit reply->finishedWithError(QNetworkReply::RemoteHostClosedError, reply->d_func()->errorString);
@@ -292,14 +297,20 @@ void QHttpNetworkConnectionChannel::receiveReply()
switch (state) {
case QHttpNetworkReplyPrivate::NothingDoneState:
case QHttpNetworkReplyPrivate::ReadingStatusState: {
+ eatWhitespace();
qint64 statusBytes = reply->d_func()->readStatus(socket);
- if (statusBytes == -1) {
- // error reading the status, close the socket and emit error
- socket->close();
+ if (statusBytes == -1 && reconnectAttempts <= 0) {
+ // too many errors reading/receiving/parsing the status, close the socket and emit error
+ close();
reply->d_func()->errorString = connection->d_func()->errorDetail(QNetworkReply::ProtocolFailure, socket);
emit reply->finishedWithError(QNetworkReply::ProtocolFailure, reply->d_func()->errorString);
QMetaObject::invokeMethod(connection, "_q_startNextRequest", Qt::QueuedConnection);
break;
+ } else if (statusBytes == -1) {
+ reconnectAttempts--;
+ reply->d_func()->clear();
+ closeAndResendCurrentRequest();
+ break;
}
bytes += statusBytes;
lastStatus = reply->d_func()->statusCode;
@@ -410,6 +421,9 @@ bool QHttpNetworkConnectionChannel::ensureConnection()
state = QHttpNetworkConnectionChannel::ConnectingState;
pendingEncrypt = connection->d_func()->encrypt;
+ // reset state
+ pipeliningSupported = PipeliningSupportUnknown;
+
// This workaround is needed since we use QAuthenticator for NTLM authentication. The "phase == Done"
// is the usual criteria for emitting authentication signals. The "phase" is set to "Done" when the
// last header for Authorization is generated by the QAuthenticator. Basic & Digest logic does not
@@ -520,6 +534,92 @@ void QHttpNetworkConnectionChannel::allDone()
// reset the reconnection attempts after we receive a complete reply.
// in case of failures, each channel will attempt two reconnects before emitting error.
reconnectAttempts = 2;
+
+ detectPipeliningSupport();
+
+ // move next from pipeline to current request
+ if (!alreadyPipelinedRequests.isEmpty()) {
+ if (resendCurrent || reply->d_func()->connectionCloseEnabled() || socket->state() != QAbstractSocket::Connected) {
+ // move the pipelined ones back to the main queue
+ requeueCurrentlyPipelinedRequests();
+ } else {
+ // there were requests pipelined in and we can continue
+ HttpMessagePair messagePair = alreadyPipelinedRequests.takeFirst();
+
+ request = messagePair.first;
+ reply = messagePair.second;
+ state = QHttpNetworkConnectionChannel::ReadingState;
+ resendCurrent = false;
+
+ written = 0; // message body, excluding the header, irrelevant here
+ bytesTotal = 0; // message body total, excluding the header, irrelevant here
+
+ // pipeline even more
+ connection->d_func()->fillPipeline(socket);
+
+ // continue reading
+ receiveReply();
+ }
+ } else if (alreadyPipelinedRequests.isEmpty() && socket->bytesAvailable() > 0) {
+ eatWhitespace();
+ // this is weird. we had nothing pipelined but still bytes available. better close it.
+ if (socket->bytesAvailable() > 0)
+ close();
+ QMetaObject::invokeMethod(connection, "_q_startNextRequest", Qt::QueuedConnection);
+ } else if (alreadyPipelinedRequests.isEmpty()) {
+ QMetaObject::invokeMethod(connection, "_q_startNextRequest", Qt::QueuedConnection);
+ }
+}
+
+void QHttpNetworkConnectionChannel::detectPipeliningSupport()
+{
+ // detect HTTP Pipelining support
+ QByteArray serverHeaderField = reply->headerField("Server");
+ if (
+ // check for broken servers in server reply header
+ // this is adapted from http://mxr.mozilla.org/firefox/ident?i=SupportsPipelining
+ (!serverHeaderField.contains("Microsoft-IIS/4."))
+ && (!serverHeaderField.contains("Microsoft-IIS/5."))
+ && (!serverHeaderField.contains("Netscape-Enterprise/3."))
+ // check for HTTP/1.1
+ && (reply->d_func()->majorVersion == 1 && reply->d_func()->minorVersion == 1)
+ // check for not having connection close
+ && (!reply->d_func()->connectionCloseEnabled())
+ // check if it is still connected
+ && (socket->state() == QAbstractSocket::Connected)
+ ) {
+ pipeliningSupported = QHttpNetworkConnectionChannel::PipeliningProbablySupported;
+ } else {
+ pipeliningSupported = QHttpNetworkConnectionChannel::PipeliningSupportUnknown;
+ }
+}
+
+// called when the connection broke and we need to queue some pipelined requests again
+void QHttpNetworkConnectionChannel::requeueCurrentlyPipelinedRequests()
+{
+ for (int i = 0; i < alreadyPipelinedRequests.length(); i++)
+ connection->d_func()->requeueRequest(alreadyPipelinedRequests.at(i));
+ alreadyPipelinedRequests.clear();
+
+ close();
+ QMetaObject::invokeMethod(connection, "_q_startNextRequest", Qt::QueuedConnection);
+}
+
+void QHttpNetworkConnectionChannel::eatWhitespace()
+{
+ char c;
+ while (socket->bytesAvailable()) {
+ if (socket->peek(&c, 1) != 1)
+ return;
+
+ // read all whitespace and line endings
+ if (c == 11 || c == '\n' || c == '\r' || c == ' ' || c == 31) {
+ socket->read(&c, 1);
+ continue;
+ } else {
+ break;
+ }
+ }
}
void QHttpNetworkConnectionChannel::handleStatus()
@@ -531,8 +631,8 @@ void QHttpNetworkConnectionChannel::handleStatus()
bool resend = false;
switch (statusCode) {
- case 401:
- case 407:
+ case 401: // auth required
+ case 407: // proxy auth required
if (connection->d_func()->handleAuthenticateChallenge(socket, reply, (statusCode == 407), resend)) {
if (resend) {
QNonContiguousByteDevice* uploadByteDevice = request.uploadByteDevice();
@@ -547,10 +647,15 @@ void QHttpNetworkConnectionChannel::handleStatus()
reply->d_func()->eraseData();
- // also use async _q_startNextRequest so we dont break with closed
- // proxy or server connections..
- resendCurrent = true;
- QMetaObject::invokeMethod(connection, "_q_startNextRequest", Qt::QueuedConnection);
+ if (alreadyPipelinedRequests.isEmpty()) {
+ // this does a re-send without closing the connection
+ resendCurrent = true;
+ QMetaObject::invokeMethod(connection, "_q_startNextRequest", Qt::QueuedConnection);
+ } else {
+ // we had requests pipelined.. better close the connection in closeAndResendCurrentRequest
+ closeAndResendCurrentRequest();
+ QMetaObject::invokeMethod(connection, "_q_startNextRequest", Qt::QueuedConnection);
+ }
}
} else {
emit reply->headerChanged();
@@ -568,26 +673,51 @@ void QHttpNetworkConnectionChannel::handleStatus()
}
}
+void QHttpNetworkConnectionChannel::pipelineInto(HttpMessagePair &pair)
+{
+ // this is only called for simple GET
+
+ QHttpNetworkRequest &request = pair.first;
+ QHttpNetworkReply *reply = pair.second;
+ if (reply) {
+ reply->d_func()->clear();
+ reply->d_func()->connection = connection;
+ reply->d_func()->autoDecompress = request.d->autoDecompress;
+ }
+#ifndef QT_NO_NETWORKPROXY
+ QByteArray header = QHttpNetworkRequestPrivate::header(request,
+ (connection->d_func()->networkProxy.type() != QNetworkProxy::NoProxy));
+#else
+ QByteArray header = QHttpNetworkRequestPrivate::header(channels[i].request,
+ false);
+#endif
+ socket->write(header);
+
+ alreadyPipelinedRequests.append(pair);
+}
+
+void QHttpNetworkConnectionChannel::closeAndResendCurrentRequest()
+{
+ requeueCurrentlyPipelinedRequests();
+ close();
+ resendCurrent = true;
+ QMetaObject::invokeMethod(connection, "_q_startNextRequest", Qt::QueuedConnection);
+}
//private slots
void QHttpNetworkConnectionChannel::_q_readyRead()
{
- if (!socket)
- return; // ### error
if (connection->d_func()->isSocketWaiting(socket) || connection->d_func()->isSocketReading(socket)) {
state = QHttpNetworkConnectionChannel::ReadingState;
if (reply)
receiveReply();
}
- // ### error
}
void QHttpNetworkConnectionChannel::_q_bytesWritten(qint64 bytes)
{
Q_UNUSED(bytes);
- if (!socket)
- return; // ### error
// bytes have been written to the socket. write even more of them :)
if (connection->d_func()->isSocketWriting(socket))
sendRequest();
@@ -596,8 +726,6 @@ void QHttpNetworkConnectionChannel::_q_bytesWritten(qint64 bytes)
void QHttpNetworkConnectionChannel::_q_disconnected()
{
- if (!socket)
- return; // ### error
// read the available data before closing
if (connection->d_func()->isSocketWaiting(socket) || connection->d_func()->isSocketReading(socket)) {
state = QHttpNetworkConnectionChannel::ReadingState;
@@ -608,17 +736,18 @@ void QHttpNetworkConnectionChannel::_q_disconnected()
QMetaObject::invokeMethod(connection, "_q_startNextRequest", Qt::QueuedConnection);
}
state = QHttpNetworkConnectionChannel::IdleState;
+
+ requeueCurrentlyPipelinedRequests();
}
void QHttpNetworkConnectionChannel::_q_connected()
{
- if (!socket)
- return; // ### error
-
// improve performance since we get the request sent by the kernel ASAP
socket->setSocketOption(QAbstractSocket::LowDelayOption, 1);
+ pipeliningSupported = QHttpNetworkConnectionChannel::PipeliningSupportUnknown;
+
// ### FIXME: if the server closes the connection unexpectedly, we shouldn't send the same broken request again!
//channels[i].reconnectAttempts = 2;
if (!pendingEncrypt) {
@@ -650,7 +779,7 @@ void QHttpNetworkConnectionChannel::_q_error(QAbstractSocket::SocketError socket
// while "Reading" the _q_disconnected() will handle this.
if (state != QHttpNetworkConnectionChannel::IdleState && state != QHttpNetworkConnectionChannel::ReadingState) {
if (reconnectAttempts-- > 0) {
- connection->d_func()->resendCurrentRequest(socket);
+ closeAndResendCurrentRequest();
return;
} else {
send2Reply = true;
@@ -663,7 +792,7 @@ void QHttpNetworkConnectionChannel::_q_error(QAbstractSocket::SocketError socket
case QAbstractSocket::SocketTimeoutError:
// try to reconnect/resend before sending an error.
if (state == QHttpNetworkConnectionChannel::WritingState && (reconnectAttempts-- > 0)) {
- connection->d_func()->resendCurrentRequest(socket);
+ closeAndResendCurrentRequest();
return;
}
send2Reply = true;
@@ -701,7 +830,7 @@ void QHttpNetworkConnectionChannel::_q_error(QAbstractSocket::SocketError socket
#ifndef QT_NO_NETWORKPROXY
void QHttpNetworkConnectionChannel::_q_proxyAuthenticationRequired(const QNetworkProxy &proxy, QAuthenticator* auth)
{
- emit connection->proxyAuthenticationRequired(proxy, auth, connection);
+ connection->d_func()->emitProxyAuthenticationRequired(this, proxy, auth);
}
#endif
diff --git a/src/network/access/qhttpnetworkconnectionchannel_p.h b/src/network/access/qhttpnetworkconnectionchannel_p.h
index 498fda9..220b72c 100644
--- a/src/network/access/qhttpnetworkconnectionchannel_p.h
+++ b/src/network/access/qhttpnetworkconnectionchannel_p.h
@@ -82,10 +82,14 @@ class QHttpNetworkReply;
class QByteArray;
class QHttpNetworkConnection;
+#ifndef HttpMessagePair
+typedef QPair<QHttpNetworkRequest, QHttpNetworkReply*> HttpMessagePair;
+#endif
+
class QHttpNetworkConnectionChannel : public QObject {
Q_OBJECT
public:
- enum ChannelState {
+ enum ChannelState {
IdleState = 0, // ready to send request
ConnectingState = 1, // connecting to host
WritingState = 2, // writing the data
@@ -112,12 +116,24 @@ public:
bool ignoreAllSslErrors;
QList<QSslError> ignoreSslErrorsList;
#endif
+
+ // HTTP pipelining -> http://en.wikipedia.org/wiki/Http_pipelining
+ enum PipeliningSupport {
+ PipeliningSupportUnknown, // default for a new connection
+ PipeliningProbablySupported, // after having received a server response that indicates support
+ PipeliningNotSupported // currently not used
+ };
+ PipeliningSupport pipeliningSupported;
+ QList<HttpMessagePair> alreadyPipelinedRequests;
+
+
QHttpNetworkConnectionChannel() : socket(0), state(IdleState), reply(0), written(0), bytesTotal(0), resendCurrent(false),
lastStatus(0), pendingEncrypt(false), reconnectAttempts(2),
authMehtod(QAuthenticatorPrivate::None), proxyAuthMehtod(QAuthenticatorPrivate::None)
#ifndef QT_NO_OPENSSL
, ignoreAllSslErrors(false)
#endif
+ , pipeliningSupported(PipeliningSupportUnknown)
, connection(0)
{}
@@ -126,13 +142,24 @@ public:
void init();
void close();
+
bool sendRequest();
void receiveReply();
+
bool ensureConnection();
+
bool expand(bool dataComplete);
void allDone(); // reply header + body have been read
void handleStatus(); // called from allDone()
+ void pipelineInto(HttpMessagePair &pair);
+ void requeueCurrentlyPipelinedRequests();
+ void detectPipeliningSupport();
+
+ void closeAndResendCurrentRequest();
+
+ void eatWhitespace();
+
protected slots:
void _q_bytesWritten(qint64 bytes); // proceed sending
void _q_readyRead(); // pending data to read
diff --git a/src/network/access/qhttpnetworkreply.cpp b/src/network/access/qhttpnetworkreply.cpp
index cd843ab..eb8ecb5 100644
--- a/src/network/access/qhttpnetworkreply.cpp
+++ b/src/network/access/qhttpnetworkreply.cpp
@@ -418,20 +418,26 @@ qint64 QHttpNetworkReplyPrivate::readStatus(QAbstractSocket *socket)
}
bool ok = parseStatus(fragment);
state = ReadingHeaderState;
- fragment.clear(); // next fragment
-
- if (!ok)
+ fragment.clear();
+ if (!ok) {
return -1;
+ }
break;
} else {
c = 0;
- bytes += socket->read(&c, 1);
+ int haveRead = socket->read(&c, 1);
+ if (haveRead == -1)
+ return -1;
+ bytes += haveRead;
fragment.append(c);
}
// is this a valid reply?
if (fragment.length() >= 5 && !fragment.startsWith("HTTP/"))
+ {
+ fragment.clear();
return -1;
+ }
}
@@ -568,7 +574,6 @@ qint64 QHttpNetworkReplyPrivate::readBodyFast(QAbstractSocket *socket, QByteData
if (contentRead + haveRead == bodyLength) {
state = AllDoneState;
- socket->readAll(); // Read the rest to clean (CRLF) ### will break pipelining
}
contentRead += haveRead;
@@ -588,8 +593,6 @@ qint64 QHttpNetworkReplyPrivate::readBody(QAbstractSocket *socket, QByteDataBuff
} else {
bytes += readReplyBodyRaw(socket, out, socket->bytesAvailable());
}
- if (state == AllDoneState)
- socket->readAll(); // Read the rest to clean (CRLF) ### will break pipelining
contentRead += bytes;
return bytes;
}
diff --git a/src/network/access/qhttpnetworkrequest.cpp b/src/network/access/qhttpnetworkrequest.cpp
index 693e1f4..8b2c4e9 100644
--- a/src/network/access/qhttpnetworkrequest.cpp
+++ b/src/network/access/qhttpnetworkrequest.cpp
@@ -49,7 +49,7 @@ QT_BEGIN_NAMESPACE
QHttpNetworkRequestPrivate::QHttpNetworkRequestPrivate(QHttpNetworkRequest::Operation op,
QHttpNetworkRequest::Priority pri, const QUrl &newUrl)
: QHttpNetworkHeaderPrivate(newUrl), operation(op), priority(pri), uploadByteDevice(0),
- autoDecompress(false)
+ autoDecompress(false), pipeliningAllowed(false)
{
}
@@ -60,6 +60,7 @@ QHttpNetworkRequestPrivate::QHttpNetworkRequestPrivate(const QHttpNetworkRequest
priority = other.priority;
uploadByteDevice = other.uploadByteDevice;
autoDecompress = other.autoDecompress;
+ pipeliningAllowed = other.pipeliningAllowed;
}
QHttpNetworkRequestPrivate::~QHttpNetworkRequestPrivate()
@@ -239,6 +240,16 @@ void QHttpNetworkRequest::setPriority(Priority priority)
d->priority = priority;
}
+bool QHttpNetworkRequest::isPipeliningAllowed() const
+{
+ return d->pipeliningAllowed;
+}
+
+void QHttpNetworkRequest::setPipeliningAllowed(bool b)
+{
+ d->pipeliningAllowed = b;
+}
+
void QHttpNetworkRequest::setUploadByteDevice(QNonContiguousByteDevice *bd)
{
d->uploadByteDevice = bd;
diff --git a/src/network/access/qhttpnetworkrequest_p.h b/src/network/access/qhttpnetworkrequest_p.h
index d4c21e5..2468be9 100644
--- a/src/network/access/qhttpnetworkrequest_p.h
+++ b/src/network/access/qhttpnetworkrequest_p.h
@@ -106,6 +106,9 @@ public:
Priority priority() const;
void setPriority(Priority priority);
+ bool isPipeliningAllowed() const;
+ void setPipeliningAllowed(bool b);
+
void setUploadByteDevice(QNonContiguousByteDevice *bd);
QNonContiguousByteDevice* uploadByteDevice() const;
@@ -133,6 +136,7 @@ public:
QHttpNetworkRequest::Priority priority;
mutable QNonContiguousByteDevice* uploadByteDevice;
bool autoDecompress;
+ bool pipeliningAllowed;
};
diff --git a/src/network/access/qnetworkaccesshttpbackend.cpp b/src/network/access/qnetworkaccesshttpbackend.cpp
index 479990a..fd47b34 100644
--- a/src/network/access/qnetworkaccesshttpbackend.cpp
+++ b/src/network/access/qnetworkaccesshttpbackend.cpp
@@ -516,6 +516,9 @@ void QNetworkAccessHttpBackend::postRequest()
return; // no need to send the request! :)
}
+ if (request().attribute(QNetworkRequest::HttpPipeliningAllowedAttribute).toBool() == true)
+ httpRequest.setPipeliningAllowed(true);
+
httpReply = http->sendRequest(httpRequest);
httpReply->setParent(this);
#ifndef QT_NO_OPENSSL
diff --git a/src/network/access/qnetworkaccessmanager.cpp b/src/network/access/qnetworkaccessmanager.cpp
index ce5f6c7..839bf31 100644
--- a/src/network/access/qnetworkaccessmanager.cpp
+++ b/src/network/access/qnetworkaccessmanager.cpp
@@ -804,7 +804,13 @@ void QNetworkAccessManagerPrivate::proxyAuthenticationRequired(QNetworkAccessBac
QAuthenticator *authenticator)
{
Q_Q(QNetworkAccessManager);
-
+ // ### FIXME Tracking of successful authentications
+ // This code is a bit broken right now for SOCKS authentication
+ // first request: proxyAuthenticationRequired gets emitted, credentials gets saved
+ // second request: (proxy != backend->reply->lastProxyAuthentication) does not evaluate to true,
+ // proxyAuthenticationRequired gets emitted again
+ // possible solution: some tracking inside the authenticator
+ // or a new function proxyAuthenticationSucceeded(true|false)
if (proxy != backend->reply->lastProxyAuthentication) {
QNetworkAuthenticationCredential *cred = fetchCachedCredentials(proxy);
if (cred) {
diff --git a/src/network/access/qnetworkrequest.cpp b/src/network/access/qnetworkrequest.cpp
index 721f8c4..7d838a3 100644
--- a/src/network/access/qnetworkrequest.cpp
+++ b/src/network/access/qnetworkrequest.cpp
@@ -167,6 +167,11 @@ QT_BEGIN_NAMESPACE
When using this flag with sequential upload data, the ContentLengthHeader
header must be set.
+ \value HttpPipeliningAllowed
+ Requests only, type: QVariant::Bool (default: false)
+ Indicates whether the QNetworkAccessManager code is
+ allowed to use HTTP pipelining with this request.
+
\value User
Special type. Additional information can be passed in
QVariants with types ranging from User to UserMax. The default
diff --git a/src/network/access/qnetworkrequest.h b/src/network/access/qnetworkrequest.h
index aaeed48..fb5ea52 100644
--- a/src/network/access/qnetworkrequest.h
+++ b/src/network/access/qnetworkrequest.h
@@ -76,6 +76,7 @@ public:
CacheSaveControlAttribute,
SourceIsFromCacheAttribute,
DoNotBufferUploadDataAttribute,
+ HttpPipeliningAllowedAttribute,
User = 1000,
UserMax = 32767
diff --git a/tests/auto/qhttpnetworkconnection/tst_qhttpnetworkconnection.cpp b/tests/auto/qhttpnetworkconnection/tst_qhttpnetworkconnection.cpp
index 4f24721..c4c33d5 100644
--- a/tests/auto/qhttpnetworkconnection/tst_qhttpnetworkconnection.cpp
+++ b/tests/auto/qhttpnetworkconnection/tst_qhttpnetworkconnection.cpp
@@ -99,6 +99,9 @@ private Q_SLOTS:
void get401_data();
void get401();
+ void getMultiple_data();
+ void getMultiple();
+ void getMultipleWithPipeliningAndMultiplePriorities();
};
@@ -763,5 +766,123 @@ void tst_QHttpNetworkConnection::nossl()
}
#endif
+
+void tst_QHttpNetworkConnection::getMultiple_data()
+{
+ QTest::addColumn<quint16>("connectionCount");
+ QTest::addColumn<bool>("pipeliningAllowed");
+ // send 100 requests. apache will usually force-close after 100 requests in a single tcp connection
+ QTest::addColumn<int>("requestCount");
+
+ QTest::newRow("6 connections, no pipelining, 100 requests") << quint16(6) << false << 100;
+ QTest::newRow("1 connection, no pipelining, 100 requests") << quint16(1) << false << 100;
+ QTest::newRow("6 connections, pipelining allowed, 100 requests") << quint16(2) << true << 100;
+ QTest::newRow("1 connection, pipelining allowed, 100 requests") << quint16(1) << true << 100;
+}
+
+void tst_QHttpNetworkConnection::getMultiple()
+{
+ QFETCH(quint16, connectionCount);
+ QFETCH(bool, pipeliningAllowed);
+ QFETCH(int, requestCount);
+
+ QHttpNetworkConnection connection(connectionCount, QtNetworkSettings::serverName());
+
+ QList<QHttpNetworkRequest*> requests;
+ QList<QHttpNetworkReply*> replies;
+
+ for (int i = 0; i < requestCount; i++) {
+ // depending on what you use the results will vary.
+ // for the "real" results, use a URL that has "internet latency" for you. Then (6 connections, pipelining) will win.
+ // for LAN latency, you will possibly get that (1 connection, no pipelining) is the fastest
+ QHttpNetworkRequest *request = new QHttpNetworkRequest("http://" + QtNetworkSettings::serverName() + "/qtest/rfc3252.txt");
+ // located in Berlin:
+ //QHttpNetworkRequest *request = new QHttpNetworkRequest(QUrl("http://klinsmann.nokia.trolltech.de/~berlin/qtcreatorad.gif"));
+ if (pipeliningAllowed)
+ request->setPipeliningAllowed(true);
+ requests.append(request);
+ QHttpNetworkReply *reply = connection.sendRequest(*request);
+ replies.append(reply);
+ }
+
+ QTime stopWatch;
+ stopWatch.start();
+ int finishedCount = 0;
+ do {
+ QCoreApplication::instance()->processEvents();
+ if (stopWatch.elapsed() >= 60000)
+ break;
+
+ finishedCount = 0;
+ for (int i = 0; i < replies.length(); i++)
+ if (replies.at(i)->isFinished())
+ finishedCount++;
+
+ } while (finishedCount != replies.length());
+
+ // redundant
+ for (int i = 0; i < replies.length(); i++)
+ QVERIFY(replies.at(i)->isFinished());
+
+ qDebug() << "===" << stopWatch.elapsed() << "msec ===";
+
+ qDeleteAll(requests);
+ qDeleteAll(replies);
+}
+
+void tst_QHttpNetworkConnection::getMultipleWithPipeliningAndMultiplePriorities()
+{
+ quint16 requestCount = 100;
+
+ // use 2 connections.
+ QHttpNetworkConnection connection(2, QtNetworkSettings::serverName());
+
+ QList<QHttpNetworkRequest*> requests;
+ QList<QHttpNetworkReply*> replies;
+
+ for (int i = 0; i < requestCount; i++) {
+
+ QHttpNetworkRequest *request = new QHttpNetworkRequest("http://" + QtNetworkSettings::serverName() + "/qtest/rfc3252.txt");
+
+ if (i % 2 || i % 3)
+ request->setPipeliningAllowed(true);
+
+ if (i % 3)
+ request->setPriority(QHttpNetworkRequest::HighPriority);
+ else if (i % 5)
+ request->setPriority(QHttpNetworkRequest::NormalPriority);
+ else if (i % 7)
+ request->setPriority(QHttpNetworkRequest::LowPriority);
+
+ requests.append(request);
+ QHttpNetworkReply *reply = connection.sendRequest(*request);
+ replies.append(reply);
+ }
+
+ QTime stopWatch;
+ stopWatch.start();
+ int finishedCount = 0;
+ do {
+ QCoreApplication::instance()->processEvents();
+ if (stopWatch.elapsed() >= 60000)
+ break;
+
+ finishedCount = 0;
+ for (int i = 0; i < replies.length(); i++)
+ if (replies.at(i)->isFinished())
+ finishedCount++;
+
+ } while (finishedCount != replies.length());
+
+ // redundant
+ for (int i = 0; i < replies.length(); i++)
+ QVERIFY(replies.at(i)->isFinished());
+
+ qDebug() << "===" << stopWatch.elapsed() << "msec ===";
+
+ qDeleteAll(requests);
+ qDeleteAll(replies);
+}
+
QTEST_MAIN(tst_QHttpNetworkConnection)
#include "tst_qhttpnetworkconnection.moc"
diff --git a/tests/auto/qnetworkreply/tst_qnetworkreply.cpp b/tests/auto/qnetworkreply/tst_qnetworkreply.cpp
index d339803..7a9d016 100644
--- a/tests/auto/qnetworkreply/tst_qnetworkreply.cpp
+++ b/tests/auto/qnetworkreply/tst_qnetworkreply.cpp
@@ -2254,8 +2254,11 @@ void tst_QNetworkReply::ioGetFromHttpBrokenServer_data()
QTest::addColumn<bool>("doDisconnect");
QTest::newRow("no-newline") << QByteArray("Hello World") << false;
- QTest::newRow("just-newline") << QByteArray("\r\n") << false;
- QTest::newRow("just-2newline") << QByteArray("\r\n\r\n") << false;
+
+ // these are OK now, we just eat the lonely newlines
+ //QTest::newRow("just-newline") << QByteArray("\r\n") << false;
+ //QTest::newRow("just-2newline") << QByteArray("\r\n\r\n") << false;
+
QTest::newRow("with-newlines") << QByteArray("Long first line\r\nLong second line") << false;
QTest::newRow("with-newlines2") << QByteArray("\r\nSecond line") << false;
QTest::newRow("with-newlines3") << QByteArray("ICY\r\nSecond line") << false;
@@ -2931,7 +2934,7 @@ void tst_QNetworkReply::ioPostToHttpFromSocket()
QSignalSpy authenticationRequiredSpy(&manager, SIGNAL(authenticationRequired(QNetworkReply*,QAuthenticator*)));
QSignalSpy proxyAuthenticationRequiredSpy(&manager, SIGNAL(proxyAuthenticationRequired(QNetworkProxy,QAuthenticator*)));
- QTestEventLoop::instance().enterLoop(1);
+ QTestEventLoop::instance().enterLoop(3);
disconnect(&manager, SIGNAL(proxyAuthenticationRequired(QNetworkProxy,QAuthenticator*)),
this, SLOT(proxyAuthenticationRequired(QNetworkProxy,QAuthenticator*)));