From f085092a48966a81315a021367086eb69c02e6a6 Mon Sep 17 00:00:00 2001 From: Markus Goetz Date: Fri, 28 Jan 2011 13:53:12 +0100 Subject: QNAM: Threaded HTTP implementation HTTP requests are run in a separate thread now. This required some big changes in the QNetworkAccessHttpBackend. There is a new class QHttpThreadDelegate which lives in the HTTP thread and is the communication layer between HTTP code and QNetworkAccessHttpBackend. Communication is done via signals/slots. The synchronous HTTP code (private QtWebKit API) also had to be completely re-worked and uses its own thread now. Reviewed-by: Peter Hartmann Task-number: QTBUG-14162 --- src/corelib/io/qnoncontiguousbytedevice.cpp | 14 +- src/corelib/io/qnoncontiguousbytedevice_p.h | 10 +- src/network/access/access.pri | 6 +- .../access/qhttpnetworkconnectionchannel.cpp | 2 +- src/network/access/qhttpnetworkreply.cpp | 6 + src/network/access/qhttpnetworkreply_p.h | 1 + src/network/access/qhttpthreaddelegate.cpp | 502 +++++++++++++ src/network/access/qhttpthreaddelegate_p.h | 283 ++++++++ src/network/access/qnetworkaccessbackend.cpp | 22 +- src/network/access/qnetworkaccessbackend_p.h | 15 +- src/network/access/qnetworkaccesshttpbackend.cpp | 795 +++++++++------------ src/network/access/qnetworkaccesshttpbackend_p.h | 43 +- src/network/access/qnetworkaccessmanager.cpp | 13 + src/network/access/qnetworkaccessmanager.h | 2 + src/network/access/qnetworkaccessmanager_p.h | 5 +- src/network/access/qnetworkreplyimpl.cpp | 57 +- src/network/access/qnetworkreplyimpl_p.h | 3 +- 17 files changed, 1272 insertions(+), 507 deletions(-) create mode 100644 src/network/access/qhttpthreaddelegate.cpp create mode 100644 src/network/access/qhttpthreaddelegate_p.h diff --git a/src/corelib/io/qnoncontiguousbytedevice.cpp b/src/corelib/io/qnoncontiguousbytedevice.cpp index 235f671..71cf92d 100644 --- a/src/corelib/io/qnoncontiguousbytedevice.cpp +++ b/src/corelib/io/qnoncontiguousbytedevice.cpp @@ -153,6 +153,7 @@ void QNonContiguousByteDevice::disableReset() resetDisabled = true; } +// FIXME we should scrap this whole implementation and instead change the ByteArrayImpl to be able to cope with sub-arrays? QNonContiguousByteDeviceBufferImpl::QNonContiguousByteDeviceBufferImpl(QBuffer *b) : QNonContiguousByteDevice() { buffer = b; @@ -244,7 +245,7 @@ qint64 QNonContiguousByteDeviceByteArrayImpl::size() return byteArray->size(); } -QNonContiguousByteDeviceRingBufferImpl::QNonContiguousByteDeviceRingBufferImpl(QRingBuffer *rb) +QNonContiguousByteDeviceRingBufferImpl::QNonContiguousByteDeviceRingBufferImpl(QSharedPointer rb) : QNonContiguousByteDevice(), currentPosition(0) { ringBuffer = rb; @@ -355,6 +356,11 @@ bool QNonContiguousByteDeviceIoDeviceImpl::advanceReadPointer(qint64 amount) // normal advancement currentReadBufferPosition += amount; + if (size() == -1) + emit readProgress(totalAdvancements, totalAdvancements); + else + emit readProgress(totalAdvancements, size()); + // advancing over that what has actually been read before if (currentReadBufferPosition > currentReadBufferAmount) { qint64 i = currentReadBufferPosition - currentReadBufferAmount; @@ -370,10 +376,6 @@ bool QNonContiguousByteDeviceIoDeviceImpl::advanceReadPointer(qint64 amount) currentReadBufferAmount = 0; } - if (size() == -1) - emit readProgress(totalAdvancements, totalAdvancements); - else - emit readProgress(totalAdvancements, size()); return true; } @@ -505,7 +507,7 @@ QNonContiguousByteDevice* QNonContiguousByteDeviceFactory::create(QIODevice *dev \internal */ -QNonContiguousByteDevice* QNonContiguousByteDeviceFactory::create(QRingBuffer *ringBuffer) +QNonContiguousByteDevice* QNonContiguousByteDeviceFactory::create(QSharedPointer ringBuffer) { return new QNonContiguousByteDeviceRingBufferImpl(ringBuffer); } diff --git a/src/corelib/io/qnoncontiguousbytedevice_p.h b/src/corelib/io/qnoncontiguousbytedevice_p.h index ff946ed..5e0b1bb 100644 --- a/src/corelib/io/qnoncontiguousbytedevice_p.h +++ b/src/corelib/io/qnoncontiguousbytedevice_p.h @@ -57,6 +57,7 @@ #include #include #include +#include #include "private/qringbuffer_p.h" QT_BEGIN_NAMESPACE @@ -70,6 +71,7 @@ public: virtual bool atEnd() = 0; virtual bool reset() = 0; void disableReset(); + bool isResetDisabled() { return resetDisabled; } virtual qint64 size() = 0; virtual ~QNonContiguousByteDevice(); @@ -89,7 +91,7 @@ class Q_CORE_EXPORT QNonContiguousByteDeviceFactory public: static QNonContiguousByteDevice* create(QIODevice *device); static QNonContiguousByteDevice* create(QByteArray *byteArray); - static QNonContiguousByteDevice* create(QRingBuffer *ringBuffer); + static QNonContiguousByteDevice* create(QSharedPointer ringBuffer); static QIODevice* wrap(QNonContiguousByteDevice* byteDevice); }; @@ -114,7 +116,7 @@ protected: class QNonContiguousByteDeviceRingBufferImpl : public QNonContiguousByteDevice { public: - QNonContiguousByteDeviceRingBufferImpl(QRingBuffer *rb); + QNonContiguousByteDeviceRingBufferImpl(QSharedPointer rb); ~QNonContiguousByteDeviceRingBufferImpl(); const char* readPointer(qint64 maximumLength, qint64 &len); bool advanceReadPointer(qint64 amount); @@ -122,13 +124,14 @@ public: bool reset(); qint64 size(); protected: - QRingBuffer* ringBuffer; + QSharedPointer ringBuffer; qint64 currentPosition; }; class QNonContiguousByteDeviceIoDeviceImpl : public QNonContiguousByteDevice { + Q_OBJECT public: QNonContiguousByteDeviceIoDeviceImpl(QIODevice *d); ~QNonContiguousByteDeviceIoDeviceImpl(); @@ -150,6 +153,7 @@ protected: class QNonContiguousByteDeviceBufferImpl : public QNonContiguousByteDevice { + Q_OBJECT public: QNonContiguousByteDeviceBufferImpl(QBuffer *b); ~QNonContiguousByteDeviceBufferImpl(); diff --git a/src/network/access/access.pri b/src/network/access/access.pri index 7497c1a..57a79b3 100644 --- a/src/network/access/access.pri +++ b/src/network/access/access.pri @@ -33,7 +33,8 @@ HEADERS += \ access/qabstractnetworkcache_p.h \ access/qabstractnetworkcache.h \ access/qnetworkdiskcache_p.h \ - access/qnetworkdiskcache.h + access/qnetworkdiskcache.h \ + access/qhttpthreaddelegate_p.h SOURCES += \ access/qftp.cpp \ @@ -60,6 +61,7 @@ SOURCES += \ access/qnetworkreplydataimpl.cpp \ access/qnetworkreplyfileimpl.cpp \ access/qabstractnetworkcache.cpp \ - access/qnetworkdiskcache.cpp + access/qnetworkdiskcache.cpp \ + access/qhttpthreaddelegate.cpp include($$PWD/../../3rdparty/zlib_dependency.pri) diff --git a/src/network/access/qhttpnetworkconnectionchannel.cpp b/src/network/access/qhttpnetworkconnectionchannel.cpp index 079f608..a26ba8a 100644 --- a/src/network/access/qhttpnetworkconnectionchannel.cpp +++ b/src/network/access/qhttpnetworkconnectionchannel.cpp @@ -253,7 +253,7 @@ bool QHttpNetworkConnectionChannel::sendRequest() #endif { // get pointer to upload data - qint64 currentReadSize; + qint64 currentReadSize = 0; qint64 desiredReadSize = qMin(socketWriteMaxSize, bytesTotal - written); const char *readPointer = uploadByteDevice->readPointer(desiredReadSize, currentReadSize); diff --git a/src/network/access/qhttpnetworkreply.cpp b/src/network/access/qhttpnetworkreply.cpp index 7f6b71f..704cf3a 100644 --- a/src/network/access/qhttpnetworkreply.cpp +++ b/src/network/access/qhttpnetworkreply.cpp @@ -177,6 +177,12 @@ qint64 QHttpNetworkReply::bytesAvailableNextBlock() const return -1; } +bool QHttpNetworkReply::readAnyAvailable() const +{ + Q_D(const QHttpNetworkReply); + return (d->responseData.bufferCount() > 0); +} + QByteArray QHttpNetworkReply::readAny() { Q_D(QHttpNetworkReply); diff --git a/src/network/access/qhttpnetworkreply_p.h b/src/network/access/qhttpnetworkreply_p.h index 1856d7a..cc0f671 100644 --- a/src/network/access/qhttpnetworkreply_p.h +++ b/src/network/access/qhttpnetworkreply_p.h @@ -125,6 +125,7 @@ public: qint64 bytesAvailable() const; qint64 bytesAvailableNextBlock() const; + bool readAnyAvailable() const; QByteArray readAny(); QByteArray readAll(); void setDownstreamLimited(bool t); diff --git a/src/network/access/qhttpthreaddelegate.cpp b/src/network/access/qhttpthreaddelegate.cpp new file mode 100644 index 0000000..d9db948 --- /dev/null +++ b/src/network/access/qhttpthreaddelegate.cpp @@ -0,0 +1,502 @@ +/**************************************************************************** +** +** Copyright (C) 2010 Nokia Corporation and/or its subsidiary(-ies). +** All rights reserved. +** Contact: Nokia Corporation (qt-info@nokia.com) +** +** This file is part of the QtNetwork module of the Qt Toolkit. +** +** $QT_BEGIN_LICENSE:LGPL$ +** No Commercial Usage +** This file contains pre-release code and may not be distributed. +** You may use this file in accordance with the terms and conditions +** contained in the Technology Preview License Agreement accompanying +** this package. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 2.1 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 2.1 requirements +** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. +** +** In addition, as a special exception, Nokia gives you certain additional +** rights. These rights are described in the Nokia Qt LGPL Exception +** version 1.1, included in the file LGPL_EXCEPTION.txt in this package. +** +** If you have questions regarding the use of this file, please contact +** Nokia at qt-info@nokia.com. +** +** +** +** +** +** +** +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#include "qhttpthreaddelegate_p.h" +#include "qthread.h" +#include "private/qnoncontiguousbytedevice_p.h" +#include + + +QT_BEGIN_NAMESPACE + +static QNetworkReply::NetworkError statusCodeFromHttp(int httpStatusCode, const QUrl &url) +{ + QNetworkReply::NetworkError code; + // we've got an error + switch (httpStatusCode) { + case 401: // Authorization required + code = QNetworkReply::AuthenticationRequiredError; + break; + + case 403: // Access denied + code = QNetworkReply::ContentOperationNotPermittedError; + break; + + case 404: // Not Found + code = QNetworkReply::ContentNotFoundError; + break; + + case 405: // Method Not Allowed + code = QNetworkReply::ContentOperationNotPermittedError; + break; + + case 407: + code = QNetworkReply::ProxyAuthenticationRequiredError; + break; + + default: + if (httpStatusCode > 500) { + // some kind of server error + code = QNetworkReply::ProtocolUnknownError; + } else if (httpStatusCode >= 400) { + // content error we did not handle above + code = QNetworkReply::UnknownContentError; + } else { + qWarning("QNetworkAccess: got HTTP status code %d which is not expected from url: \"%s\"", + httpStatusCode, qPrintable(url.toString())); + code = QNetworkReply::ProtocolFailure; + } + } + + return code; +} + + +static QByteArray makeCacheKey(QUrl &url, QNetworkProxy *proxy) +{ + QByteArray result; + QUrl copy = url; + bool isEncrypted = copy.scheme().toLower() == QLatin1String("https"); + copy.setPort(copy.port(isEncrypted ? 443 : 80)); + result = copy.toEncoded(QUrl::RemoveUserInfo | QUrl::RemovePath | + QUrl::RemoveQuery | QUrl::RemoveFragment); + +#ifndef QT_NO_NETWORKPROXY + if (proxy && proxy->type() != QNetworkProxy::NoProxy) { + QUrl key; + + switch (proxy->type()) { + case QNetworkProxy::Socks5Proxy: + key.setScheme(QLatin1String("proxy-socks5")); + break; + + case QNetworkProxy::HttpProxy: + case QNetworkProxy::HttpCachingProxy: + key.setScheme(QLatin1String("proxy-http")); + break; + + default: + break; + } + + if (!key.scheme().isEmpty()) { + key.setUserName(proxy->user()); + key.setHost(proxy->hostName()); + key.setPort(proxy->port()); + key.setEncodedQuery(result); + result = key.toEncoded(); + } + } +#endif + + return "http-connection:" + result; +} + +class QNetworkAccessCachedHttpConnection: public QHttpNetworkConnection, + public QNetworkAccessCache::CacheableObject +{ + // Q_OBJECT +public: + QNetworkAccessCachedHttpConnection(const QString &hostName, quint16 port, bool encrypt) + : QHttpNetworkConnection(hostName, port, encrypt) + { + setExpires(true); + setShareable(true); + } + + virtual void dispose() + { +#if 0 // sample code; do this right with the API + Q_ASSERT(!isWorking()); +#endif + delete this; + } +}; + + +QThreadStorage QHttpThreadDelegate::connections; + + +QHttpThreadDelegate::~QHttpThreadDelegate() +{ + // It could be that the main thread has asked us to shut down, so we need to delete the HTTP reply + if (httpReply) { + delete httpReply; + } + + // Get the object cache that stores our QHttpNetworkConnection objects + // and release the entry for this QHttpNetworkConnection + if (connections.hasLocalData() && !cacheKey.isEmpty()) { + connections.localData()->releaseEntry(cacheKey); + } +} + + +QHttpThreadDelegate::QHttpThreadDelegate(QObject *parent) : + QObject(parent) + , ssl(false) + , downloadBufferMaximumSize(0) + , pendingDownloadData(0) + , pendingDownloadProgress(0) + , synchronous(false) + , incomingStatusCode(0) + , isPipeliningUsed(false) + , incomingContentLength(-1) + , incomingErrorCode(QNetworkReply::NoError) + , downloadBuffer(0) + , httpConnection(0) + , httpReply(0) +{ +} + +// This is invoked as BlockingQueuedConnection from QNetworkAccessHttpBackend in the user thread +void QHttpThreadDelegate::startRequestSynchronously() +{ + synchronous = true; + + QEventLoop synchronousRequestLoop; + this->synchronousRequestLoop = &synchronousRequestLoop; + + // Worst case timeout + QTimer::singleShot(30*1000, this, SLOT(abortRequest())); + + QMetaObject::invokeMethod(this, "startRequest", Qt::QueuedConnection); + synchronousRequestLoop.exec(); + + connections.localData()->releaseEntry(cacheKey); + connections.setLocalData(0); + +} + + +// This is invoked as QueuedConnection from QNetworkAccessHttpBackend in the user thread +void QHttpThreadDelegate::startRequest() +{ + // Check QThreadStorage for the QNetworkAccessCache + // If not there, create this connection cache + if (!connections.hasLocalData()) { + connections.setLocalData(new QNetworkAccessCache()); + } + + // check if we have an open connection to this host + QUrl urlCopy = httpRequest.url(); + urlCopy.setPort(urlCopy.port(ssl ? 443 : 80)); + + if (transparentProxy.type() != QNetworkProxy::NoProxy) + cacheKey = makeCacheKey(urlCopy, &transparentProxy); + else if (cacheProxy.type() != QNetworkProxy::NoProxy) + cacheKey = makeCacheKey(urlCopy, &cacheProxy); + else + cacheKey = makeCacheKey(urlCopy, 0); + + + // the http object is actually a QHttpNetworkConnection + httpConnection = static_cast(connections.localData()->requestEntryNow(cacheKey)); + if (httpConnection == 0) { + // no entry in cache; create an object + // the http object is actually a QHttpNetworkConnection + httpConnection = new QNetworkAccessCachedHttpConnection(urlCopy.host(), urlCopy.port(), ssl); +#ifndef QT_NO_OPENSSL + // Set the QSslConfiguration from this QNetworkRequest. + if (ssl) { + httpConnection->setSslConfiguration(incomingSslConfiguration); + } +#endif + +#ifndef QT_NO_NETWORKPROXY + httpConnection->setTransparentProxy(transparentProxy); + httpConnection->setCacheProxy(cacheProxy); +#endif + + // cache the QHttpNetworkConnection corresponding to this cache key + connections.localData()->addEntry(cacheKey, httpConnection); + } + + + // Send the request to the connection + httpReply = httpConnection->sendRequest(httpRequest); + httpReply->setParent(this); + + // Connect the reply signals that we need to handle and then forward + if (synchronous) { + connect(httpReply,SIGNAL(headerChanged()), this, SLOT(synchronousHeaderChangedSlot())); + connect(httpReply,SIGNAL(finished()), this, SLOT(synchronousFinishedSlot())); + connect(httpReply,SIGNAL(finishedWithError(QNetworkReply::NetworkError, const QString)), + this, SLOT(synchronousFinishedWithErrorSlot(QNetworkReply::NetworkError,QString))); + + connect(httpReply, SIGNAL(authenticationRequired(QHttpNetworkRequest,QAuthenticator*)), + this, SLOT(synchronousAuthenticationRequiredSlot(QHttpNetworkRequest,QAuthenticator*))); + connect(httpReply, SIGNAL(proxyAuthenticationRequired(QNetworkProxy,QAuthenticator*)), + this, SLOT(synchronousProxyAuthenticationRequiredSlot(QNetworkProxy,QAuthenticator*))); + + // Don't care about ignored SSL errors for now in the synchronous HTTP case. + } else if (!synchronous) { + connect(httpReply,SIGNAL(headerChanged()), this, SLOT(headerChangedSlot())); + connect(httpReply,SIGNAL(finished()), this, SLOT(finishedSlot())); + connect(httpReply,SIGNAL(finishedWithError(QNetworkReply::NetworkError, const QString)), + this, SLOT(finishedWithErrorSlot(QNetworkReply::NetworkError,QString))); + // some signals are only interesting when normal asynchronous style is used + connect(httpReply,SIGNAL(readyRead()), this, SLOT(readyReadSlot())); + connect(httpReply,SIGNAL(dataReadProgress(int, int)), this, SLOT(dataReadProgressSlot(int,int))); + connect(httpReply, SIGNAL(cacheCredentials(QHttpNetworkRequest,QAuthenticator*)), + this, SLOT(cacheCredentialsSlot(QHttpNetworkRequest,QAuthenticator*))); +#ifndef QT_NO_OPENSSL + connect(httpReply,SIGNAL(sslErrors(const QList)), this, SLOT(sslErrorsSlot(QList))); +#endif + + // In the asynchronous HTTP case we can just forward those signals + // Connect the reply signals that we can directly forward + connect(httpReply, SIGNAL(authenticationRequired(QHttpNetworkRequest,QAuthenticator*)), + this, SIGNAL(authenticationRequired(QHttpNetworkRequest,QAuthenticator*))); + connect(httpReply, SIGNAL(proxyAuthenticationRequired(QNetworkProxy,QAuthenticator*)), + this, SIGNAL(proxyAuthenticationRequired(QNetworkProxy,QAuthenticator*))); + } +} + +// This gets called from the user thread or by the synchronous HTTP timeout timer +void QHttpThreadDelegate::abortRequest() +{ + if (httpReply) { + delete httpReply; + httpReply = 0; + this->deleteLater(); + } + + // Got aborted by the timeout timer + if (synchronous) + incomingErrorCode = QNetworkReply::TimeoutError; +} + +void QHttpThreadDelegate::readyReadSlot() +{ + // Don't do in zerocopy case + if (!downloadBuffer.isNull()) + return; + + while (httpReply->readAnyAvailable()) { + pendingDownloadData->fetchAndAddRelease(1); + emit downloadData(httpReply->readAny()); + } +} + +void QHttpThreadDelegate::finishedSlot() +{ + // If there is still some data left emit that now + while (httpReply->readAnyAvailable()) { + pendingDownloadData->fetchAndAddRelease(1); + emit downloadData(httpReply->readAny()); + } + +#ifndef QT_NO_OPENSSL + if (ssl) + emit sslConfigurationChanged(httpReply->sslConfiguration()); +#endif + + if (httpReply->statusCode() >= 400) { + // it's an error reply + QString msg = QLatin1String(QT_TRANSLATE_NOOP("QNetworkReply", + "Error downloading %1 - server replied: %2")); + msg = msg.arg(QString::fromAscii(httpRequest.url().toEncoded()), httpReply->reasonPhrase()); + emit error(statusCodeFromHttp(httpReply->statusCode(), httpRequest.url()), msg); + } + + emit downloadFinished(); + + QMetaObject::invokeMethod(httpReply, "deleteLater", Qt::QueuedConnection); + QMetaObject::invokeMethod(this, "deleteLater", Qt::QueuedConnection); + httpReply = 0; +} + +void QHttpThreadDelegate::synchronousFinishedSlot() +{ + if (httpReply->statusCode() >= 400) { + // it's an error reply + QString msg = QLatin1String(QT_TRANSLATE_NOOP("QNetworkReply", + "Error downloading %1 - server replied: %2")); + incomingErrorDetail = msg.arg(QString::fromAscii(httpRequest.url().toEncoded()), httpReply->reasonPhrase()); + incomingErrorCode = statusCodeFromHttp(httpReply->statusCode(), httpRequest.url()); + } + + synchronousDownloadData = httpReply->readAll(); + + QMetaObject::invokeMethod(httpReply, "deleteLater", Qt::QueuedConnection); + QMetaObject::invokeMethod(synchronousRequestLoop, "quit", Qt::QueuedConnection); + httpReply = 0; +} + +void QHttpThreadDelegate::finishedWithErrorSlot(QNetworkReply::NetworkError errorCode, const QString &detail) +{ +#ifndef QT_NO_OPENSSL + if (ssl) + emit sslConfigurationChanged(httpReply->sslConfiguration()); +#endif + emit error(errorCode,detail); + emit downloadFinished(); + + + QMetaObject::invokeMethod(httpReply, "deleteLater", Qt::QueuedConnection); + QMetaObject::invokeMethod(this, "deleteLater", Qt::QueuedConnection); + httpReply = 0; +} + + +void QHttpThreadDelegate::synchronousFinishedWithErrorSlot(QNetworkReply::NetworkError errorCode, const QString &detail) +{ + incomingErrorCode = errorCode; + incomingErrorDetail = detail; + + QMetaObject::invokeMethod(httpReply, "deleteLater", Qt::QueuedConnection); + QMetaObject::invokeMethod(synchronousRequestLoop, "quit", Qt::QueuedConnection); + httpReply = 0; +} + +static void downloadBufferDeleter(char *ptr) +{ + delete[] ptr; +} + +void QHttpThreadDelegate::headerChangedSlot() +{ +#ifndef QT_NO_OPENSSL + if (ssl) + emit sslConfigurationChanged(httpReply->sslConfiguration()); +#endif + + // Is using a zerocopy buffer allowed by user and possible with this reply? + if (httpReply->supportsUserProvidedDownloadBuffer() + && downloadBufferMaximumSize > 0) { + char *buf = new char[httpReply->contentLength()]; // throws if allocation fails + if (buf) { + downloadBuffer = QSharedPointer(buf, downloadBufferDeleter); + httpReply->setUserProvidedDownloadBuffer(buf); + } + } + + // We fetch this into our own + incomingHeaders = httpReply->header(); + incomingStatusCode = httpReply->statusCode(); + incomingReasonPhrase = httpReply->reasonPhrase(); + isPipeliningUsed = httpReply->isPipeliningUsed(); + incomingContentLength = httpReply->contentLength(); + + emit downloadMetaData(incomingHeaders, + incomingStatusCode, + incomingReasonPhrase, + isPipeliningUsed, + downloadBuffer, + incomingContentLength); +} + +void QHttpThreadDelegate::synchronousHeaderChangedSlot() +{ + // Store the information we need in this object, the QNetworkAccessHttpBackend will later read it + incomingHeaders = httpReply->header(); + incomingStatusCode = httpReply->statusCode(); + incomingReasonPhrase = httpReply->reasonPhrase(); + isPipeliningUsed = httpReply->isPipeliningUsed(); + incomingContentLength = httpReply->contentLength(); +} + + +void QHttpThreadDelegate::dataReadProgressSlot(int done, int total) +{ + // If we don't have a download buffer don't attempt to go this codepath + // It is not used by QNetworkAccessHttpBackend + if (downloadBuffer.isNull()) + return; + + pendingDownloadProgress->fetchAndAddRelease(1); + emit downloadProgress(done, total); +} + +void QHttpThreadDelegate::cacheCredentialsSlot(const QHttpNetworkRequest &request, QAuthenticator *authenticator) +{ + authenticationManager->cacheCredentials(request.url(), authenticator); +} + + +#ifndef QT_NO_OPENSSL +void QHttpThreadDelegate::sslErrorsSlot(const QList &errors) +{ + emit sslConfigurationChanged(httpReply->sslConfiguration()); + + bool ignoreAll = false; + QList specificErrors; + emit sslErrors(errors, &ignoreAll, &specificErrors); + if (ignoreAll) + httpReply->ignoreSslErrors(); + if (!specificErrors.isEmpty()) + httpReply->ignoreSslErrors(specificErrors); +} +#endif + +void QHttpThreadDelegate::synchronousAuthenticationRequiredSlot(const QHttpNetworkRequest &request, QAuthenticator *a) +{ + Q_UNUSED(request); + + // Ask the credential cache + QNetworkAuthenticationCredential credential = authenticationManager->fetchCachedCredentials(httpRequest.url(), a); + if (!credential.isNull()) { + a->setUser(credential.user); + a->setPassword(credential.password); + } + + // Disconnect this connection now since we only want to ask the authentication cache once. + QObject::disconnect(this, SLOT(synchronousAuthenticationRequiredSlot(QHttpNetworkRequest,QAuthenticator*))); +} + +#ifndef QT_NO_NETWORKPROXY +void QHttpThreadDelegate::synchronousProxyAuthenticationRequiredSlot(const QNetworkProxy &p, QAuthenticator *a) +{ + // Ask the credential cache + QNetworkAuthenticationCredential credential = authenticationManager->fetchCachedProxyCredentials(p, a); + if (!credential.isNull()) { + a->setUser(credential.user); + a->setPassword(credential.password); + } + + // Disconnect this connection now since we only want to ask the authentication cache once. + QObject::disconnect(this, SLOT(synchronousProxyAuthenticationRequiredSlot(QNetworkProxy,QAuthenticator*))); +} + +#endif + +QT_END_NAMESPACE diff --git a/src/network/access/qhttpthreaddelegate_p.h b/src/network/access/qhttpthreaddelegate_p.h new file mode 100644 index 0000000..7ec8efb --- /dev/null +++ b/src/network/access/qhttpthreaddelegate_p.h @@ -0,0 +1,283 @@ +/**************************************************************************** +** +** Copyright (C) 2010 Nokia Corporation and/or its subsidiary(-ies). +** All rights reserved. +** Contact: Nokia Corporation (qt-info@nokia.com) +** +** This file is part of the QtNetwork module of the Qt Toolkit. +** +** $QT_BEGIN_LICENSE:LGPL$ +** No Commercial Usage +** This file contains pre-release code and may not be distributed. +** You may use this file in accordance with the terms and conditions +** contained in the Technology Preview License Agreement accompanying +** this package. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 2.1 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 2.1 requirements +** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. +** +** In addition, as a special exception, Nokia gives you certain additional +** rights. These rights are described in the Nokia Qt LGPL Exception +** version 1.1, included in the file LGPL_EXCEPTION.txt in this package. +** +** If you have questions regarding the use of this file, please contact +** Nokia at qt-info@nokia.com. +** +** +** +** +** +** +** +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#ifndef QHTTPTHREADDELEGATE_H +#define QHTTPTHREADDELEGATE_H + + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists for the convenience +// of the Network Access API. This header file may change from +// version to version without notice, or even be removed. +// +// We mean it. +// + +#include +#include +#include +#include +#include +#include +#include +#include +#include "qnetworkaccesscache_p.h" +#include "qhttpnetworkrequest_p.h" +#include "qhttpnetworkconnection_p.h" +#include "qhttpnetworkreply_p.h" +#include "QSharedPointer" +#include "qsslconfiguration.h" +#include "private/qnoncontiguousbytedevice_p.h" +#include "qnetworkaccessauthenticationmanager_p.h" + +QT_BEGIN_NAMESPACE + +class QNetworkAccessCachedHttpConnection; +class QHttpThreadDelegate : public QObject +{ + Q_OBJECT +public: + explicit QHttpThreadDelegate(QObject *parent = 0); + + ~QHttpThreadDelegate(); + + // incoming + bool ssl; +#ifndef QT_NO_OPENSSL + QSslConfiguration incomingSslConfiguration; +#endif + QHttpNetworkRequest httpRequest; + qint64 downloadBufferMaximumSize; + // From backend, modified by us for signal compression + QSharedPointer pendingDownloadData; + QSharedPointer pendingDownloadProgress; +#ifndef QT_NO_NETWORKPROXY + QNetworkProxy cacheProxy; + QNetworkProxy transparentProxy; +#endif + QSharedPointer authenticationManager; + bool synchronous; + + // outgoing, Retrieved in the synchronous HTTP case + QByteArray synchronousDownloadData; + QList > incomingHeaders; + int incomingStatusCode; + QString incomingReasonPhrase; + bool isPipeliningUsed; + qint64 incomingContentLength; + QNetworkReply::NetworkError incomingErrorCode; + QString incomingErrorDetail; + +protected: + // The zerocopy download buffer, if used: + QSharedPointer downloadBuffer; + // The QHttpNetworkConnection that is used + QNetworkAccessCachedHttpConnection *httpConnection; + QByteArray cacheKey; + QHttpNetworkReply *httpReply; + + // Used for implementing the synchronous HTTP, see startRequestSynchronously() + QEventLoop *synchronousRequestLoop; + +signals: + void authenticationRequired(const QHttpNetworkRequest &request, QAuthenticator *); +#ifndef QT_NO_NETWORKPROXY + void proxyAuthenticationRequired(const QNetworkProxy &, QAuthenticator *); +#endif +#ifndef QT_NO_OPENSSL + void sslErrors(const QList &, bool *, QList *); + void sslConfigurationChanged(const QSslConfiguration); +#endif + void downloadMetaData(QList >,int,QString,bool,QSharedPointer,qint64); + void downloadProgress(qint64, qint64); + void downloadData(QByteArray); + void error(QNetworkReply::NetworkError, const QString); + void downloadFinished(); +public slots: + // This are called via QueuedConnection from user thread + void startRequest(); + void abortRequest(); + // This is called with a BlockingQueuedConnection from user thread + void startRequestSynchronously(); +protected slots: + // From QHttp* + void readyReadSlot(); + void finishedSlot(); + void finishedWithErrorSlot(QNetworkReply::NetworkError errorCode, const QString &detail = QString()); + void synchronousFinishedSlot(); + void synchronousFinishedWithErrorSlot(QNetworkReply::NetworkError errorCode, const QString &detail = QString()); + void headerChangedSlot(); + void synchronousHeaderChangedSlot(); + void dataReadProgressSlot(int done, int total); + void cacheCredentialsSlot(const QHttpNetworkRequest &request, QAuthenticator *authenticator); +#ifndef QT_NO_OPENSSL + void sslErrorsSlot(const QList &errors); +#endif + + void synchronousAuthenticationRequiredSlot(const QHttpNetworkRequest &request, QAuthenticator *); +#ifndef QT_NO_NETWORKPROXY + void synchronousProxyAuthenticationRequiredSlot(const QNetworkProxy &, QAuthenticator *); +#endif + +protected: + // Cache for all the QHttpNetworkConnection objects. + // This is per thread. + static QThreadStorage connections; + +}; + +// This QNonContiguousByteDevice is connected to the QNetworkAccessHttpBackend +// and represents the PUT/POST data. +class QNonContiguousByteDeviceThreadForwardImpl : public QNonContiguousByteDevice +{ + Q_OBJECT +protected: + bool wantDataPending; + qint64 m_amount; + char *m_data; + QByteArray m_dataArray; + bool m_atEnd; + qint64 m_size; +public: + QNonContiguousByteDeviceThreadForwardImpl(bool aE, qint64 s) + : QNonContiguousByteDevice(), + wantDataPending(false), + m_amount(0), + m_data(0), + m_atEnd(aE), + m_size(s) + { + } + + ~QNonContiguousByteDeviceThreadForwardImpl() + { + } + + const char* readPointer(qint64 maximumLength, qint64 &len) + { + if (m_amount == 0 && wantDataPending == false) { + len = 0; + wantDataPending = true; + emit wantData(maximumLength); + } else if (m_amount == 0 && wantDataPending == true) { + // Do nothing, we already sent a wantData signal and wait for results + len = 0; + } else if (m_amount > 0) { + len = m_amount; + return m_data; + } + // cannot happen + return 0; + } + + bool advanceReadPointer(qint64 a) + { + if (m_data == 0) + return false; + + m_amount -= a; + m_data += a; + + // To main thread to inform about our state + emit processedData(a); + + // FIXME possible optimization, already ask user thread for some data + + return true; + } + + bool atEnd() + { + if (m_amount > 0) + return false; + else + return m_atEnd; + } + + bool reset() + { + m_amount = 0; + m_data = 0; + + // Communicate as BlockingQueuedConnection + bool b = false; + emit resetData(&b); + return b; + } + + qint64 size() + { + return m_size; + } + +public slots: + // From user thread: + void haveDataSlot(QByteArray dataArray, bool dataAtEnd, qint64 dataSize) + { + wantDataPending = false; + + m_dataArray = dataArray; + m_data = const_cast(m_dataArray.constData()); + m_amount = dataArray.size(); + + m_atEnd = dataAtEnd; + m_size = dataSize; + + // This will tell the HTTP code (QHttpNetworkConnectionChannel) that we have data available now + emit readyRead(); + } + +signals: + // void readyRead(); in parent class + // void readProgress(qint64 current, qint64 total); happens in the main thread with the real bytedevice + + // to main thread: + void wantData(qint64); + void processedData(qint64); + void resetData(bool *b); +}; + +QT_END_NAMESPACE + +#endif // QHTTPTHREADDELEGATE_H diff --git a/src/network/access/qnetworkaccessbackend.cpp b/src/network/access/qnetworkaccessbackend.cpp index 53ae29e..5aedac9 100644 --- a/src/network/access/qnetworkaccessbackend.cpp +++ b/src/network/access/qnetworkaccessbackend.cpp @@ -106,12 +106,10 @@ QNetworkAccessBackend *QNetworkAccessManagerPrivate::findBackend(QNetworkAccessM QNonContiguousByteDevice* QNetworkAccessBackend::createUploadByteDevice() { - QNonContiguousByteDevice* device = 0; - if (reply->outgoingDataBuffer) - device = QNonContiguousByteDeviceFactory::create(reply->outgoingDataBuffer); + uploadByteDevice = QSharedPointer(QNonContiguousByteDeviceFactory::create(reply->outgoingDataBuffer)); else if (reply->outgoingData) { - device = QNonContiguousByteDeviceFactory::create(reply->outgoingData); + uploadByteDevice = QSharedPointer(QNonContiguousByteDeviceFactory::create(reply->outgoingData)); } else { return 0; } @@ -120,14 +118,13 @@ QNonContiguousByteDevice* QNetworkAccessBackend::createUploadByteDevice() reply->request.attribute(QNetworkRequest::DoNotBufferUploadDataAttribute, QVariant(false)) == QVariant(true); if (bufferDisallowed) - device->disableReset(); - - // make sure we delete this later - device->setParent(this); + uploadByteDevice->disableReset(); - connect(device, SIGNAL(readProgress(qint64,qint64)), this, SLOT(emitReplyUploadProgress(qint64,qint64))); + // We want signal emissions only for normal asynchronous uploads + if (!isSynchronous()) + connect(uploadByteDevice.data(), SIGNAL(readProgress(qint64,qint64)), this, SLOT(emitReplyUploadProgress(qint64,qint64))); - return device; + return uploadByteDevice.data(); } // need to have this function since the reply is a private member variable @@ -327,11 +324,6 @@ void QNetworkAccessBackend::authenticationRequired(QAuthenticator *authenticator manager->authenticationRequired(this, authenticator); } -void QNetworkAccessBackend::cacheCredentials(QAuthenticator *authenticator) -{ - manager->authenticationManager->cacheCredentials(this->reply->url, authenticator); -} - void QNetworkAccessBackend::metaDataChanged() { reply->metaDataChanged(); diff --git a/src/network/access/qnetworkaccessbackend_p.h b/src/network/access/qnetworkaccessbackend_p.h index 26ee61a..644ae2d 100644 --- a/src/network/access/qnetworkaccessbackend_p.h +++ b/src/network/access/qnetworkaccessbackend_p.h @@ -70,7 +70,6 @@ class QNetworkAccessManagerPrivate; class QNetworkReplyImplPrivate; class QAbstractNetworkCache; class QNetworkCacheMetaData; -class QNetworkAccessBackendUploadIODevice; class QNonContiguousByteDevice; // Should support direct file upload from disk or download to disk. @@ -175,7 +174,6 @@ protected: // Create the device used for reading the upload data QNonContiguousByteDevice* createUploadByteDevice(); - // these functions control the downstream mechanism // that is, data that has come via the connection and is going out the backend qint64 nextDownstreamBlockSize() const; @@ -185,6 +183,8 @@ protected: void writeDownstreamDataDownloadBuffer(qint64, qint64); char* getDownloadBuffer(qint64); + QSharedPointer uploadByteDevice; + public slots: // for task 251801, needs to be a slot to be called asynchronously void writeDownstreamData(QIODevice *data); @@ -196,19 +196,22 @@ protected slots: void proxyAuthenticationRequired(const QNetworkProxy &proxy, QAuthenticator *auth); #endif void authenticationRequired(QAuthenticator *auth); - void cacheCredentials(QAuthenticator *auth); void metaDataChanged(); void redirectionRequested(const QUrl &destination); void sslErrors(const QList &errors); void emitReplyUploadProgress(qint64 bytesSent, qint64 bytesTotal); +protected: + // FIXME In the long run we should get rid of our QNAM architecture + // and scrap this ReplyImpl/Backend distinction. + QNetworkAccessManagerPrivate *manager; + QNetworkReplyImplPrivate *reply; + private: friend class QNetworkAccessManager; friend class QNetworkAccessManagerPrivate; - friend class QNetworkAccessBackendUploadIODevice; friend class QNetworkReplyImplPrivate; - QNetworkAccessManagerPrivate *manager; - QNetworkReplyImplPrivate *reply; + bool synchronous; }; diff --git a/src/network/access/qnetworkaccesshttpbackend.cpp b/src/network/access/qnetworkaccesshttpbackend.cpp index 79c43fa..4908e0a 100644 --- a/src/network/access/qnetworkaccesshttpbackend.cpp +++ b/src/network/access/qnetworkaccesshttpbackend.cpp @@ -52,60 +52,19 @@ #include "QtCore/qdatetime.h" #include "QtCore/qelapsedtimer.h" #include "QtNetwork/qsslconfiguration.h" +#include "qhttpthreaddelegate_p.h" +#include "qthread.h" #ifndef QT_NO_HTTP #include // for strchr -QT_BEGIN_NAMESPACE +Q_DECLARE_METATYPE(QSharedPointer) -enum { - DefaultHttpPort = 80, - DefaultHttpsPort = 443 -}; +QT_BEGIN_NAMESPACE class QNetworkProxy; -static QByteArray makeCacheKey(QNetworkAccessHttpBackend *backend, QNetworkProxy *proxy) -{ - QByteArray result; - QUrl copy = backend->url(); - bool isEncrypted = copy.scheme().toLower() == QLatin1String("https"); - copy.setPort(copy.port(isEncrypted ? DefaultHttpsPort : DefaultHttpPort)); - result = copy.toEncoded(QUrl::RemoveUserInfo | QUrl::RemovePath | - QUrl::RemoveQuery | QUrl::RemoveFragment); - -#ifndef QT_NO_NETWORKPROXY - if (proxy->type() != QNetworkProxy::NoProxy) { - QUrl key; - - switch (proxy->type()) { - case QNetworkProxy::Socks5Proxy: - key.setScheme(QLatin1String("proxy-socks5")); - break; - - case QNetworkProxy::HttpProxy: - case QNetworkProxy::HttpCachingProxy: - key.setScheme(QLatin1String("proxy-http")); - break; - - default: - break; - } - - if (!key.scheme().isEmpty()) { - key.setUserName(proxy->user()); - key.setHost(proxy->hostName()); - key.setPort(proxy->port()); - key.setEncodedQuery(result); - result = key.toEncoded(); - } - } -#endif - - return "http-connection:" + result; -} - static inline bool isSeparator(register char c) { static const char separators[] = "()<>@,;:\\\"/[]?={}"; @@ -230,71 +189,13 @@ QNetworkAccessHttpBackendFactory::create(QNetworkAccessManager::Operation op, return 0; } -static QNetworkReply::NetworkError statusCodeFromHttp(int httpStatusCode, const QUrl &url) -{ - QNetworkReply::NetworkError code; - // we've got an error - switch (httpStatusCode) { - case 401: // Authorization required - code = QNetworkReply::AuthenticationRequiredError; - break; - - case 403: // Access denied - code = QNetworkReply::ContentOperationNotPermittedError; - break; - - case 404: // Not Found - code = QNetworkReply::ContentNotFoundError; - break; - - case 405: // Method Not Allowed - code = QNetworkReply::ContentOperationNotPermittedError; - break; - - case 407: - code = QNetworkReply::ProxyAuthenticationRequiredError; - break; - - default: - if (httpStatusCode > 500) { - // some kind of server error - code = QNetworkReply::ProtocolUnknownError; - } else if (httpStatusCode >= 400) { - // content error we did not handle above - code = QNetworkReply::UnknownContentError; - } else { - qWarning("QNetworkAccess: got HTTP status code %d which is not expected from url: \"%s\"", - httpStatusCode, qPrintable(url.toString())); - code = QNetworkReply::ProtocolFailure; - } - } - - return code; -} - -class QNetworkAccessCachedHttpConnection: public QHttpNetworkConnection, - public QNetworkAccessCache::CacheableObject -{ - // Q_OBJECT -public: - QNetworkAccessCachedHttpConnection(const QString &hostName, quint16 port, bool encrypt) - : QHttpNetworkConnection(hostName, port, encrypt) - { - setExpires(true); - setShareable(true); - } - - virtual void dispose() - { -#if 0 // sample code; do this right with the API - Q_ASSERT(!isWorking()); -#endif - delete this; - } -}; - QNetworkAccessHttpBackend::QNetworkAccessHttpBackend() - : QNetworkAccessBackend(), httpReply(0), http(0), uploadDevice(0) + : QNetworkAccessBackend() + , statusCode(0) + , pendingDownloadDataEmissions(new QAtomicInt()) + , pendingDownloadProgressEmissions(new QAtomicInt()) + , loadingFromCache(false) + , usingZerocopyDownloadBuffer(false) #ifndef QT_NO_OPENSSL , pendingSslConfiguration(0), pendingIgnoreAllSslErrors(false) #endif @@ -304,44 +205,14 @@ QNetworkAccessHttpBackend::QNetworkAccessHttpBackend() QNetworkAccessHttpBackend::~QNetworkAccessHttpBackend() { - if (http) - disconnectFromHttp(); + // This will do nothing if the request was already finished or aborted + emit abortHttpRequest(); + #ifndef QT_NO_OPENSSL delete pendingSslConfiguration; #endif } -void QNetworkAccessHttpBackend::disconnectFromHttp() -{ - if (http) { - // This is abut disconnecting signals, not about disconnecting TCP connections - disconnect(http, 0, this, 0); - - // Get the object cache that stores our QHttpNetworkConnection objects - QNetworkAccessCache *cache = QNetworkAccessManagerPrivate::getObjectCache(this); - - // synchronous calls are not put into the cache, so for them the key is empty - if (!cacheKey.isEmpty()) - cache->releaseEntry(cacheKey); - } - - // This is abut disconnecting signals, not about disconnecting TCP connections - if (httpReply) - disconnect(httpReply, 0, this, 0); - - http = 0; - httpReply = 0; - cacheKey.clear(); -} - -void QNetworkAccessHttpBackend::finished() -{ - if (http) - disconnectFromHttp(); - // call parent - QNetworkAccessBackend::finished(); -} - /* For a given httpRequest 1) If AlwaysNetwork, return @@ -487,9 +358,82 @@ static QHttpNetworkRequest::Priority convert(const QNetworkRequest::Priority& pr void QNetworkAccessHttpBackend::postRequest() { + QThread *thread = 0; + if (isSynchronous()) { + // A synchronous HTTP request uses its own thread + thread = new QThread(); + QObject::connect(thread, SIGNAL(finished()), thread, SLOT(deleteLater())); + thread->start(); + } else if (!manager->httpThread) { + // We use the manager-global thread. + // At some point we could switch to having multiple threads if it makes sense. + manager->httpThread = new QThread(); + QObject::connect(manager->httpThread, SIGNAL(finished()), manager->httpThread, SLOT(deleteLater())); + manager->httpThread->start(); +#ifndef QT_NO_NETWORKPROXY + qRegisterMetaType("QNetworkProxy"); +#endif +#ifndef QT_NO_OPENSSL + qRegisterMetaType >("QList"); + qRegisterMetaType("QSslConfiguration"); +#endif + qRegisterMetaType > >("QList >"); + qRegisterMetaType("QHttpNetworkRequest"); + qRegisterMetaType("QNetworkReply::NetworkError"); + qRegisterMetaType >("QSharedPointer"); + + thread = manager->httpThread; + } else { + // Asynchronous request, thread already exists + thread = manager->httpThread; + } + + QUrl url = request().url(); + httpRequest.setUrl(url); + + bool ssl = url.scheme().toLower() == QLatin1String("https"); + setAttribute(QNetworkRequest::ConnectionEncryptedAttribute, ssl); + httpRequest.setSsl(ssl); + + +#ifndef QT_NO_NETWORKPROXY + QNetworkProxy transparentProxy, cacheProxy; + + foreach (const QNetworkProxy &p, proxyList()) { + // use the first proxy that works + // for non-encrypted connections, any transparent or HTTP proxy + // for encrypted, only transparent proxies + if (!ssl + && (p.capabilities() & QNetworkProxy::CachingCapability) + && (p.type() == QNetworkProxy::HttpProxy || + p.type() == QNetworkProxy::HttpCachingProxy)) { + cacheProxy = p; + transparentProxy = QNetworkProxy::NoProxy; + break; + } + if (p.isTransparentProxy()) { + transparentProxy = p; + cacheProxy = QNetworkProxy::NoProxy; + break; + } + } + + // check if at least one of the proxies + if (transparentProxy.type() == QNetworkProxy::DefaultProxy && + cacheProxy.type() == QNetworkProxy::DefaultProxy) { + // unsuitable proxies + QMetaObject::invokeMethod(this, "error", isSynchronous() ? Qt::DirectConnection : Qt::QueuedConnection, + Q_ARG(QNetworkReply::NetworkError, QNetworkReply::ProxyNotFoundError), + Q_ARG(QString, tr("No suitable proxy found"))); + QMetaObject::invokeMethod(this, "finished", isSynchronous() ? Qt::DirectConnection : Qt::QueuedConnection); + return; + } +#endif + + bool loadedFromCache = false; - QHttpNetworkRequest httpRequest; httpRequest.setPriority(convert(request().priority())); + switch (operation()) { case QNetworkAccessManager::GetOperation: httpRequest.setOperation(QHttpNetworkRequest::Get); @@ -504,13 +448,13 @@ void QNetworkAccessHttpBackend::postRequest() case QNetworkAccessManager::PostOperation: invalidateCache(); httpRequest.setOperation(QHttpNetworkRequest::Post); - httpRequest.setUploadByteDevice(createUploadByteDevice()); + createUploadByteDevice(); break; case QNetworkAccessManager::PutOperation: invalidateCache(); httpRequest.setOperation(QHttpNetworkRequest::Put); - httpRequest.setUploadByteDevice(createUploadByteDevice()); + createUploadByteDevice(); break; case QNetworkAccessManager::DeleteOperation: @@ -521,7 +465,7 @@ void QNetworkAccessHttpBackend::postRequest() case QNetworkAccessManager::CustomOperation: invalidateCache(); // for safety reasons, we don't know what the operation does httpRequest.setOperation(QHttpNetworkRequest::Custom); - httpRequest.setUploadByteDevice(createUploadByteDevice()); + createUploadByteDevice(); httpRequest.setCustomVerb(request().attribute( QNetworkRequest::CustomVerbAttribute).toByteArray()); break; @@ -530,11 +474,6 @@ void QNetworkAccessHttpBackend::postRequest() break; // can't happen } - bool encrypt = (url().scheme().toLower() == QLatin1String("https")); - httpRequest.setSsl(encrypt); - - httpRequest.setUrl(url()); - QList headers = request().rawHeaderList(); if (resumeOffset != 0) { if (headers.contains("Range")) { @@ -558,6 +497,7 @@ void QNetworkAccessHttpBackend::postRequest() httpRequest.setHeaderField("Range", "bytes=" + QByteArray::number(resumeOffset) + '-'); } } + foreach (const QByteArray &header, headers) httpRequest.setHeaderField(header, request().rawHeader(header)); @@ -576,30 +516,155 @@ void QNetworkAccessHttpBackend::postRequest() QNetworkRequest::Automatic).toInt()) == QNetworkRequest::Manual) httpRequest.setWithCredentials(false); - httpReply = http->sendRequest(httpRequest); - httpReply->setParent(this); + + // Create the HTTP thread delegate + QHttpThreadDelegate *delegate = new QHttpThreadDelegate; + + // For the synchronous HTTP, this is the normal way the delegate gets deleted + // For the asynchronous HTTP this is a safety measure, the delegate deletes itself when HTTP is finished + connect(thread, SIGNAL(finished()), delegate, SLOT(deleteLater())); + + // Set the properties it needs + delegate->httpRequest = httpRequest; +#ifndef QT_NO_NETWORKPROXY + delegate->cacheProxy = cacheProxy; + delegate->transparentProxy = transparentProxy; +#endif + delegate->ssl = ssl; #ifndef QT_NO_OPENSSL - if (pendingSslConfiguration) - httpReply->setSslConfiguration(*pendingSslConfiguration); - if (pendingIgnoreAllSslErrors) - httpReply->ignoreSslErrors(); - httpReply->ignoreSslErrors(pendingIgnoreSslErrorsList); - connect(httpReply, SIGNAL(sslErrors(QList)), - SLOT(sslErrors(QList))); + if (ssl) + delegate->incomingSslConfiguration = request().sslConfiguration(); #endif - connect(httpReply, SIGNAL(finished()), SLOT(replyFinished())); - connect(httpReply, SIGNAL(finishedWithError(QNetworkReply::NetworkError,QString)), - SLOT(httpError(QNetworkReply::NetworkError,QString))); - connect(httpReply, SIGNAL(headerChanged()), SLOT(replyHeaderChanged())); - connect(httpReply, SIGNAL(cacheCredentials(QHttpNetworkRequest,QAuthenticator*)), - SLOT(httpCacheCredentials(QHttpNetworkRequest,QAuthenticator*))); + // Do we use synchronous HTTP? + delegate->synchronous = isSynchronous(); + + // The authentication manager is used to avoid the BlockingQueuedConnection communication + // from HTTP thread to user thread in some cases. + delegate->authenticationManager = manager->authenticationManager; + + if (!isSynchronous()) { + // Tell our zerocopy policy to the delegate + delegate->downloadBufferMaximumSize = + request().attribute(QNetworkRequest::MaximumDownloadBufferSizeAttribute).toLongLong(); + + // These atomic integers are used for signal compression + delegate->pendingDownloadData = pendingDownloadDataEmissions; + delegate->pendingDownloadProgress = pendingDownloadProgressEmissions; + + // Connect the signals of the delegate to us + connect(delegate, SIGNAL(downloadData(QByteArray)), + this, SLOT(replyDownloadData(QByteArray)), + Qt::QueuedConnection); + connect(delegate, SIGNAL(downloadFinished()), + this, SLOT(replyFinished()), + Qt::QueuedConnection); + connect(delegate, SIGNAL(downloadMetaData(QList >,int,QString,bool,QSharedPointer,qint64)), + this, SLOT(replyDownloadMetaData(QList >,int,QString,bool,QSharedPointer,qint64)), + Qt::QueuedConnection); + connect(delegate, SIGNAL(downloadProgress(qint64,qint64)), + this, SLOT(replyDownloadProgressSlot(qint64,qint64)), + Qt::QueuedConnection); + connect(delegate, SIGNAL(error(QNetworkReply::NetworkError,QString)), + this, SLOT(httpError(QNetworkReply::NetworkError, const QString)), + Qt::QueuedConnection); +#ifndef QT_NO_OPENSSL + connect(delegate, SIGNAL(sslConfigurationChanged(QSslConfiguration)), + this, SLOT(replySslConfigurationChanged(QSslConfiguration)), + Qt::QueuedConnection); +#endif + // Those need to report back, therefire BlockingQueuedConnection + connect(delegate, SIGNAL(authenticationRequired(QHttpNetworkRequest,QAuthenticator*)), + this, SLOT(httpAuthenticationRequired(QHttpNetworkRequest,QAuthenticator*)), + Qt::BlockingQueuedConnection); #ifndef QT_NO_NETWORKPROXY - connect(httpReply, SIGNAL(proxyAuthenticationRequired(QNetworkProxy,QAuthenticator*)), - SLOT(proxyAuthenticationRequired(QNetworkProxy,QAuthenticator*))); + connect (delegate, SIGNAL(proxyAuthenticationRequired(QNetworkProxy,QAuthenticator*)), + this, SLOT(proxyAuthenticationRequired(QNetworkProxy,QAuthenticator*)), + Qt::BlockingQueuedConnection); #endif - connect(httpReply, SIGNAL(authenticationRequired(const QHttpNetworkRequest,QAuthenticator*)), - SLOT(httpAuthenticationRequired(const QHttpNetworkRequest,QAuthenticator*))); +#ifndef QT_NO_OPENSSL + connect(delegate, SIGNAL(sslErrors(QList,bool*,QList*)), + this, SLOT(replySslErrors(const QList &, bool *, QList *)), + Qt::BlockingQueuedConnection); +#endif + // This signal we will use to start the request. + connect(this, SIGNAL(startHttpRequest()), delegate, SLOT(startRequest())); + connect(this, SIGNAL(abortHttpRequest()), delegate, SLOT(abortRequest())); + + if (uploadByteDevice) { + QNonContiguousByteDeviceThreadForwardImpl *forwardUploadDevice = + new QNonContiguousByteDeviceThreadForwardImpl(uploadByteDevice->atEnd(), uploadByteDevice->size()); + if (uploadByteDevice->isResetDisabled()) + forwardUploadDevice->disableReset(); + forwardUploadDevice->setParent(delegate); // needed to make sure it is moved on moveToThread() + delegate->httpRequest.setUploadByteDevice(forwardUploadDevice); + + // From main thread to user thread: + QObject::connect(this, SIGNAL(haveUploadData(QByteArray, bool, qint64)), + forwardUploadDevice, SLOT(haveDataSlot(QByteArray, bool, qint64)), Qt::QueuedConnection); + QObject::connect(uploadByteDevice.data(), SIGNAL(readyRead()), + forwardUploadDevice, SIGNAL(readyRead()), + Qt::QueuedConnection); + + // From http thread to user thread: + QObject::connect(forwardUploadDevice, SIGNAL(wantData(qint64)), + this, SLOT(wantUploadDataSlot(qint64))); + QObject::connect(forwardUploadDevice, SIGNAL(processedData(qint64)), + this, SLOT(sentUploadDataSlot(qint64))); + connect(forwardUploadDevice, SIGNAL(resetData(bool*)), + this, SLOT(resetUploadDataSlot(bool*)), + Qt::BlockingQueuedConnection); // this is the only one with BlockingQueued! + } + } else if (isSynchronous()) { + connect(this, SIGNAL(startHttpRequestSynchronously()), delegate, SLOT(startRequestSynchronously()), Qt::BlockingQueuedConnection); + + if (uploadByteDevice) { + // For the synchronous HTTP use case the use thread (this one here) is blocked + // so we cannot use the asynchronous upload architecture. + // We therefore won't use the QNonContiguousByteDeviceThreadForwardImpl but directly + // use the uploadByteDevice provided to us by the QNetworkReplyImpl. + // The code that is in QNetworkReplyImplPrivate::setup() makes sure it is safe to use from a thread + // since it only wraps a QRingBuffer + delegate->httpRequest.setUploadByteDevice(uploadByteDevice.data()); + } + } + + + // Move the delegate to the http thread + delegate->moveToThread(thread); + // This call automatically moves the uploadDevice too for the asynchronous case. + + // Send an signal to the delegate so it starts working in the other thread + if (isSynchronous()) { + emit startHttpRequestSynchronously(); // This one is BlockingQueuedConnection, so it will return when all work is done + + if (delegate->incomingErrorCode != QNetworkReply::NoError) { + replyDownloadMetaData + (delegate->incomingHeaders, + delegate->incomingStatusCode, + delegate->incomingReasonPhrase, + delegate->isPipeliningUsed, + QSharedPointer(), + delegate->incomingContentLength); + httpError(delegate->incomingErrorCode, delegate->incomingErrorDetail); + } else { + replyDownloadMetaData + (delegate->incomingHeaders, + delegate->incomingStatusCode, + delegate->incomingReasonPhrase, + delegate->isPipeliningUsed, + QSharedPointer(), + delegate->incomingContentLength); + replyDownloadData(delegate->synchronousDownloadData); + } + + // End the thread. It will delete itself from the finished() signal + thread->quit(); + + finished(); + } else { + emit startHttpRequest(); // Signal to the HTTP thread and go back to user. + } } void QNetworkAccessHttpBackend::invalidateCache() @@ -611,159 +676,52 @@ void QNetworkAccessHttpBackend::invalidateCache() void QNetworkAccessHttpBackend::open() { - QUrl url = request().url(); - bool encrypt = url.scheme().toLower() == QLatin1String("https"); - setAttribute(QNetworkRequest::ConnectionEncryptedAttribute, encrypt); - - // set the port number in the reply if it wasn't set - url.setPort(url.port(encrypt ? DefaultHttpsPort : DefaultHttpPort)); - - QNetworkProxy *theProxy = 0; -#ifndef QT_NO_NETWORKPROXY - QNetworkProxy transparentProxy, cacheProxy; - - foreach (const QNetworkProxy &p, proxyList()) { - // use the first proxy that works - // for non-encrypted connections, any transparent or HTTP proxy - // for encrypted, only transparent proxies - if (!encrypt - && (p.capabilities() & QNetworkProxy::CachingCapability) - && (p.type() == QNetworkProxy::HttpProxy || - p.type() == QNetworkProxy::HttpCachingProxy)) { - cacheProxy = p; - transparentProxy = QNetworkProxy::NoProxy; - theProxy = &cacheProxy; - break; - } - if (p.isTransparentProxy()) { - transparentProxy = p; - cacheProxy = QNetworkProxy::NoProxy; - theProxy = &transparentProxy; - break; - } - } - - // check if at least one of the proxies - if (transparentProxy.type() == QNetworkProxy::DefaultProxy && - cacheProxy.type() == QNetworkProxy::DefaultProxy) { - // unsuitable proxies - if (isSynchronous()) { - error(QNetworkReply::ProxyNotFoundError, tr("No suitable proxy found")); - finished(); - } else { - QMetaObject::invokeMethod(this, "error", Qt::QueuedConnection, - Q_ARG(QNetworkReply::NetworkError, QNetworkReply::ProxyNotFoundError), - Q_ARG(QString, tr("No suitable proxy found"))); - QMetaObject::invokeMethod(this, "finished", Qt::QueuedConnection); - } - return; - } -#endif - - if (isSynchronous()) { - // for synchronous requests, we just create a new connection - http = new QHttpNetworkConnection(1, url.host(), url.port(), encrypt, this); -#ifndef QT_NO_NETWORKPROXY - http->setTransparentProxy(transparentProxy); - http->setCacheProxy(cacheProxy); -#endif - postRequest(); - processRequestSynchronously(); - } else { - // check if we have an open connection to this host - cacheKey = makeCacheKey(this, theProxy); - QNetworkAccessCache *cache = QNetworkAccessManagerPrivate::getObjectCache(this); - // the http object is actually a QHttpNetworkConnection - http = static_cast(cache->requestEntryNow(cacheKey)); - if (http == 0) { - // no entry in cache; create an object - // the http object is actually a QHttpNetworkConnection - http = new QNetworkAccessCachedHttpConnection(url.host(), url.port(), encrypt); - -#ifndef QT_NO_NETWORKPROXY - http->setTransparentProxy(transparentProxy); - http->setCacheProxy(cacheProxy); -#endif - - // cache the QHttpNetworkConnection corresponding to this cache key - cache->addEntry(cacheKey, static_cast(http.data())); - } - postRequest(); - } + postRequest(); } void QNetworkAccessHttpBackend::closeDownstreamChannel() { - // this indicates that the user closed the stream while the reply isn't finished yet + // FIXME Maybe we can get rid of this whole architecture part } void QNetworkAccessHttpBackend::downstreamReadyWrite() { - readFromHttp(); - if (httpReply && httpReply->bytesAvailable() == 0 && httpReply->isFinished()) - replyFinished(); + // FIXME Maybe we can get rid of this whole architecture part } void QNetworkAccessHttpBackend::setDownstreamLimited(bool b) { - if (httpReply) - httpReply->setDownstreamLimited(b); -} - -void QNetworkAccessHttpBackend::replyReadyRead() -{ - readFromHttp(); + Q_UNUSED(b); + // We know that readBuffer maximum size limiting is broken since quite a while. + // The task to fix this is QTBUG-15065 } -void QNetworkAccessHttpBackend::readFromHttp() +void QNetworkAccessHttpBackend::replyDownloadData(QByteArray d) { - if (!httpReply) + int pendingSignals = (int)pendingDownloadDataEmissions->fetchAndAddAcquire(-1) - 1; + + if (pendingSignals > 0) { + // Some more signal emissions to this slot are pending. + // Instead of writing the downstream data, we wait + // and do it in the next call we get + // (signal comppression) + pendingDownloadData.append(d); return; - - // We read possibly more than nextDownstreamBlockSize(), but - // this is not a critical thing since it is already in the - // memory anyway - - QByteDataBuffer list; - - while (httpReply->bytesAvailable() != 0 && nextDownstreamBlockSize() != 0 && nextDownstreamBlockSize() > list.byteAmount()) { - list.append(httpReply->readAny()); } - if (!list.isEmpty()) - writeDownstreamData(list); + pendingDownloadData.append(d); + d.clear(); + writeDownstreamData(pendingDownloadData); + pendingDownloadData.clear(); } void QNetworkAccessHttpBackend::replyFinished() { - if (httpReply->bytesAvailable()) - // we haven't read everything yet. Wait some more. + // We are already loading from cache, we still however + // got this signal because it was posted already + if (loadingFromCache) return; - int statusCode = httpReply->statusCode(); - if (statusCode >= 400) { - // it's an error reply - QString msg = QLatin1String(QT_TRANSLATE_NOOP("QNetworkReply", - "Error downloading %1 - server replied: %2")); - msg = msg.arg(url().toString(), httpReply->reasonPhrase()); - error(statusCodeFromHttp(httpReply->statusCode(), httpReply->url()), msg); - } - -#ifndef QT_NO_OPENSSL - // store the SSL configuration now - // once we call finished(), we won't have access to httpReply anymore - QSslConfiguration sslConfig = httpReply->sslConfiguration(); - if (pendingSslConfiguration) { - *pendingSslConfiguration = sslConfig; - } else if (!sslConfig.isNull()) { - QT_TRY { - pendingSslConfiguration = new QSslConfiguration(sslConfig); - } QT_CATCH(...) { - qWarning("QNetworkAccess: could not allocate a QSslConfiguration object for a SSL connection."); - } - } -#endif - finished(); } @@ -785,12 +743,27 @@ void QNetworkAccessHttpBackend::checkForRedirect(const int statusCode) } } -void QNetworkAccessHttpBackend::replyHeaderChanged() +void QNetworkAccessHttpBackend::replyDownloadMetaData + (QList > hm, + int sc,QString rp,bool pu, + QSharedPointer db, + qint64 contentLength) { - setAttribute(QNetworkRequest::HttpPipeliningWasUsedAttribute, httpReply->isPipeliningUsed()); + statusCode = sc; + reasonPhrase = rp; + + // Download buffer + if (!db.isNull()) { + reply->setDownloadBuffer(db, contentLength); + usingZerocopyDownloadBuffer = true; + } else { + usingZerocopyDownloadBuffer = false; + } + + setAttribute(QNetworkRequest::HttpPipeliningWasUsedAttribute, pu); // reconstruct the HTTP header - QList > headerMap = httpReply->header(); + QList > headerMap = hm; QList >::ConstIterator it = headerMap.constBegin(), end = headerMap.constEnd(); QByteArray header; @@ -807,11 +780,10 @@ void QNetworkAccessHttpBackend::replyHeaderChanged() setRawHeader(it->first, value); } - setAttribute(QNetworkRequest::HttpStatusCodeAttribute, httpReply->statusCode()); - setAttribute(QNetworkRequest::HttpReasonPhraseAttribute, httpReply->reasonPhrase()); + setAttribute(QNetworkRequest::HttpStatusCodeAttribute, statusCode); + setAttribute(QNetworkRequest::HttpReasonPhraseAttribute, reasonPhrase); // is it a redirection? - const int statusCode = httpReply->statusCode(); checkForRedirect(statusCode); if (statusCode >= 500 && statusCode < 600) { @@ -854,29 +826,21 @@ void QNetworkAccessHttpBackend::replyHeaderChanged() setCachingEnabled(true); } - // Check if a download buffer is supported from the HTTP reply - char *buf = 0; - if (httpReply->supportsUserProvidedDownloadBuffer()) { - // Check if a download buffer is supported by the user - buf = getDownloadBuffer(httpReply->contentLength()); - if (buf) { - httpReply->setUserProvidedDownloadBuffer(buf); - // If there is a download buffer we react on the progress signal - connect(httpReply, SIGNAL(dataReadProgress(int,int)), SLOT(replyDownloadProgressSlot(int,int))); - } - } - - // If there is no buffer, we react on the readyRead signal - if (!buf) { - connect(httpReply, SIGNAL(readyRead()), SLOT(replyReadyRead())); - } - metaDataChanged(); } -void QNetworkAccessHttpBackend::replyDownloadProgressSlot(int received, int total) +void QNetworkAccessHttpBackend::replyDownloadProgressSlot(qint64 received, qint64 total) { // we can be sure here that there is a download buffer + + int pendingSignals = (int)pendingDownloadProgressEmissions->fetchAndAddAcquire(-1) - 1; + if (pendingSignals > 0) { + // Let's ignore this signal and look at the next one coming in + // (signal comppression) + return; + } + + // Now do the actual notification of new bytes writeDownstreamDataDownloadBuffer(received, total); } @@ -886,20 +850,62 @@ void QNetworkAccessHttpBackend::httpAuthenticationRequired(const QHttpNetworkReq authenticationRequired(auth); } -void QNetworkAccessHttpBackend::httpCacheCredentials(const QHttpNetworkRequest &, - QAuthenticator *auth) -{ - cacheCredentials(auth); -} - void QNetworkAccessHttpBackend::httpError(QNetworkReply::NetworkError errorCode, const QString &errorString) { #if defined(QNETWORKACCESSHTTPBACKEND_DEBUG) qDebug() << "http error!" << errorCode << errorString; #endif + error(errorCode, errorString); - finished(); +} + +#ifndef QT_NO_OPENSSL +void QNetworkAccessHttpBackend::replySslErrors( + const QList &list, bool *ignoreAll, QList *toBeIgnored) +{ + // Go to generic backend + sslErrors(list); + // Check if the callback set any ignore and return this here to http thread + if (pendingIgnoreAllSslErrors) + *ignoreAll = true; + if (!pendingIgnoreSslErrorsList.isEmpty()) + *toBeIgnored = pendingIgnoreSslErrorsList; +} + +void QNetworkAccessHttpBackend::replySslConfigurationChanged(const QSslConfiguration &c) +{ + // Receiving the used SSL configuration from the HTTP thread + if (pendingSslConfiguration) + *pendingSslConfiguration = c; + else if (!c.isNull()) + pendingSslConfiguration = new QSslConfiguration(c); +} +#endif + +// Coming from QNonContiguousByteDeviceThreadForwardImpl in HTTP thread +void QNetworkAccessHttpBackend::resetUploadDataSlot(bool *r) +{ + *r = uploadByteDevice->reset(); +} + +// Coming from QNonContiguousByteDeviceThreadForwardImpl in HTTP thread +void QNetworkAccessHttpBackend::sentUploadDataSlot(qint64 amount) +{ + uploadByteDevice->advanceReadPointer(amount); +} + +// Coming from QNonContiguousByteDeviceThreadForwardImpl in HTTP thread +void QNetworkAccessHttpBackend::wantUploadDataSlot(qint64 maxSize) +{ + // call readPointer + qint64 currentUploadDataLength = 0; + char *data = const_cast(uploadByteDevice->readPointer(maxSize, currentUploadDataLength)); + // Let's make a copy of this data + QByteArray dataArray(data, currentUploadDataLength); + + // Communicate back to HTTP thread + emit haveUploadData(dataArray, uploadByteDevice->atEnd(), uploadByteDevice->size()); } /* @@ -950,8 +956,10 @@ bool QNetworkAccessHttpBackend::sendCacheContents(const QNetworkCacheMetaData &m #if defined(QNETWORKACCESSHTTPBACKEND_DEBUG) qDebug() << "Successfully sent cache:" << url() << contents->size() << "bytes"; #endif - if (httpReply) - disconnect(httpReply, SIGNAL(finished()), this, SLOT(replyFinished())); + + // Set the following flag so we can ignore some signals from HTTP thread + // that would still come + loadingFromCache = true; return true; } @@ -964,39 +972,29 @@ void QNetworkAccessHttpBackend::copyFinished(QIODevice *dev) #ifndef QT_NO_OPENSSL void QNetworkAccessHttpBackend::ignoreSslErrors() { - if (httpReply) - httpReply->ignoreSslErrors(); - else - pendingIgnoreAllSslErrors = true; + pendingIgnoreAllSslErrors = true; } void QNetworkAccessHttpBackend::ignoreSslErrors(const QList &errors) { - if (httpReply) { - httpReply->ignoreSslErrors(errors); - } else { - // the pending list is set if QNetworkReply::ignoreSslErrors(const QList &errors) - // is called before QNetworkAccessManager::get() (or post(), etc.) - pendingIgnoreSslErrorsList = errors; - } + // the pending list is set if QNetworkReply::ignoreSslErrors(const QList &errors) + // is called before QNetworkAccessManager::get() (or post(), etc.) + pendingIgnoreSslErrorsList = errors; } void QNetworkAccessHttpBackend::fetchSslConfiguration(QSslConfiguration &config) const { - if (httpReply) - config = httpReply->sslConfiguration(); - else if (pendingSslConfiguration) + if (pendingSslConfiguration) config = *pendingSslConfiguration; + else + config = request().sslConfiguration(); } void QNetworkAccessHttpBackend::setSslConfiguration(const QSslConfiguration &newconfig) { - if (httpReply) - httpReply->setSslConfiguration(newconfig); - else if (pendingSslConfiguration) - *pendingSslConfiguration = newconfig; - else - pendingSslConfiguration = new QSslConfiguration(newconfig); + // Setting a SSL configuration on a reply is not supported. The user needs to set + // her/his QSslConfiguration on the QNetworkRequest. + Q_UNUSED(newconfig); } #endif @@ -1101,7 +1099,7 @@ QNetworkCacheMetaData QNetworkAccessHttpBackend::fetchCacheMetaData(const QNetwo bool canDiskCache; // only cache GET replies by default, all other replies (POST, PUT, DELETE) // are not cacheable by default (according to RFC 2616 section 9) - if (httpReply->request().operation() == QHttpNetworkRequest::Get) { + if (httpRequest.operation() == QHttpNetworkRequest::Get) { canDiskCache = true; // 14.32 @@ -1119,7 +1117,7 @@ QNetworkCacheMetaData QNetworkAccessHttpBackend::fetchCacheMetaData(const QNetwo canDiskCache = false; // responses to POST might be cacheable - } else if (httpReply->request().operation() == QHttpNetworkRequest::Post) { + } else if (httpRequest.operation() == QHttpNetworkRequest::Post) { canDiskCache = false; // some pages contain "expires:" and "cache-control: no-cache" field, @@ -1133,12 +1131,11 @@ QNetworkCacheMetaData QNetworkAccessHttpBackend::fetchCacheMetaData(const QNetwo } metaData.setSaveToDisk(canDiskCache); - int statusCode = httpReply->statusCode(); QNetworkCacheMetaData::AttributesMap attributes; if (statusCode != 304) { // update the status code attributes.insert(QNetworkRequest::HttpStatusCodeAttribute, statusCode); - attributes.insert(QNetworkRequest::HttpReasonPhraseAttribute, httpReply->reasonPhrase()); + attributes.insert(QNetworkRequest::HttpReasonPhraseAttribute, reasonPhrase); } else { // this is a redirection, keep the attributes intact attributes = oldMetaData.attributes(); @@ -1154,7 +1151,8 @@ bool QNetworkAccessHttpBackend::canResume() const return false; // Can only resume if server/resource supports Range header. - if (httpReply->headerField("Accept-Ranges", "none") == "none") + QByteArray acceptRangesheaderName("Accept-Ranges"); + if (!hasRawHeader(acceptRangesheaderName) || rawHeader(acceptRangesheaderName) == "none") return false; // We only support resuming for byte ranges. @@ -1166,7 +1164,7 @@ bool QNetworkAccessHttpBackend::canResume() const // If we're using a download buffer then we don't support resuming/migration // right now. Too much trouble. - if (httpReply->userProvidedDownloadBuffer()) + if (usingZerocopyDownloadBuffer) return false; return true; @@ -1177,87 +1175,6 @@ void QNetworkAccessHttpBackend::setResumeOffset(quint64 offset) resumeOffset = offset; } -bool QNetworkAccessHttpBackend::processRequestSynchronously() -{ - QHttpNetworkConnectionChannel *channel = &http->channels()[0]; - - // Disconnect all socket signals. They will only confuse us when using waitFor* - QObject::disconnect(channel->socket, 0, 0, 0); - - qint64 timeout = 20*1000; // 20 sec - QElapsedTimer timeoutTimer; - - bool waitResult = channel->socket->waitForConnected(timeout); - timeoutTimer.start(); - - if (!waitResult || channel->socket->state() != QAbstractSocket::ConnectedState) { - error(QNetworkReply::UnknownNetworkError, QLatin1String("could not connect")); - return false; - } - channel->_q_connected(); // this will send the request (via sendRequest()) - -#ifndef QT_NO_OPENSSL - if (http->isSsl()) { - qint64 remainingTimeEncrypted = timeout - timeoutTimer.elapsed(); - if (!static_cast(channel->socket)->waitForEncrypted(remainingTimeEncrypted)) { - error(QNetworkReply::SslHandshakeFailedError, - QLatin1String("could not encrypt or timeout while encrypting")); - return false; - } - channel->_q_encrypted(); - } -#endif - - // if we get a 401 or 407, we might need to send the request twice, see below - bool authenticating = false; - - do { - channel->sendRequest(); - - qint64 remainingTimeBytesWritten; - while(channel->socket->bytesToWrite() > 0 || - channel->state == QHttpNetworkConnectionChannel::WritingState) { - remainingTimeBytesWritten = timeout - timeoutTimer.elapsed(); - channel->sendRequest(); // triggers channel->socket->write() - if (!channel->socket->waitForBytesWritten(remainingTimeBytesWritten)) { - error(QNetworkReply::TimeoutError, - QLatin1String("could not write bytes to socket or timeout while writing")); - return false; - } - } - - qint64 remainingTimeBytesRead = timeout - timeoutTimer.elapsed(); - // Loop for at most remainingTime until either the socket disconnects - // or the reply is finished - do { - waitResult = channel->socket->waitForReadyRead(remainingTimeBytesRead); - remainingTimeBytesRead = timeout - timeoutTimer.elapsed(); - if (!waitResult || remainingTimeBytesRead <= 0 - || channel->socket->state() != QAbstractSocket::ConnectedState) { - error(QNetworkReply::TimeoutError, - QLatin1String("could not read from socket or timeout while reading")); - return false; - } - - if (channel->socket->bytesAvailable()) - channel->_q_readyRead(); - - if (!httpReply) - return false; // we got a 401 or 407 and cannot handle it (it might happen that - // disconnectFromHttp() was called, in that case the reply is zero) - // ### I am quite sure this does not work for NTLM - // ### how about uploading to an auth / proxyAuth site? - - authenticating = (httpReply->statusCode() == 401 || httpReply->statusCode() == 407); - - if (httpReply->isFinished()) - break; - } while (remainingTimeBytesRead > 0); - } while (authenticating); - - return true; -} - QT_END_NAMESPACE #endif // QT_NO_HTTP diff --git a/src/network/access/qnetworkaccesshttpbackend_p.h b/src/network/access/qnetworkaccesshttpbackend_p.h index 7064d4a..712dd2f 100644 --- a/src/network/access/qnetworkaccesshttpbackend_p.h +++ b/src/network/access/qnetworkaccesshttpbackend_p.h @@ -61,6 +61,8 @@ #include "QtCore/qpointer.h" #include "QtCore/qdatetime.h" +#include "QtCore/qsharedpointer.h" +#include "qatomic.h" #ifndef QT_NO_HTTP @@ -99,24 +101,44 @@ public: bool canResume() const; void setResumeOffset(quint64 offset); - virtual bool processRequestSynchronously(); +signals: + // To HTTP thread: + void startHttpRequest(); + void abortHttpRequest(); + void startHttpRequestSynchronously(); + + void haveUploadData(QByteArray dataArray, bool dataAtEnd, qint64 dataSize); private slots: - void replyReadyRead(); + // From HTTP thread: + void replyDownloadData(QByteArray); void replyFinished(); - void replyHeaderChanged(); - void replyDownloadProgressSlot(int,int); + void replyDownloadMetaData(QList >,int,QString,bool,QSharedPointer,qint64); + void replyDownloadProgressSlot(qint64,qint64); void httpAuthenticationRequired(const QHttpNetworkRequest &request, QAuthenticator *auth); - void httpCacheCredentials(const QHttpNetworkRequest &request, QAuthenticator *auth); void httpError(QNetworkReply::NetworkError error, const QString &errorString); +#ifndef QT_NO_OPENSSL + void replySslErrors(const QList &, bool *, QList *); + void replySslConfigurationChanged(const QSslConfiguration&); +#endif + + // From QNonContiguousByteDeviceThreadForwardImpl in HTTP thread: + void resetUploadDataSlot(bool *r); + void wantUploadDataSlot(qint64); + void sentUploadDataSlot(qint64); + bool sendCacheContents(const QNetworkCacheMetaData &metaData); - void finished(); // override private: - QHttpNetworkReply *httpReply; - QPointer http; - QByteArray cacheKey; - QNetworkAccessBackendUploadIODevice *uploadDevice; + QHttpNetworkRequest httpRequest; // There is also a copy in the HTTP thread + int statusCode; + QString reasonPhrase; + // Will be increased by HTTP thread: + QSharedPointer pendingDownloadDataEmissions; + QSharedPointer pendingDownloadProgressEmissions; + bool loadingFromCache; + QByteDataBuffer pendingDownloadData; + bool usingZerocopyDownloadBuffer; #ifndef QT_NO_OPENSSL QSslConfiguration *pendingSslConfiguration; @@ -126,7 +148,6 @@ private: quint64 resumeOffset; - void disconnectFromHttp(); void validateCache(QHttpNetworkRequest &httpRequest, bool &loadedFromCache); void invalidateCache(); void postRequest(); diff --git a/src/network/access/qnetworkaccessmanager.cpp b/src/network/access/qnetworkaccessmanager.cpp index 562141b..ca8ea0e 100644 --- a/src/network/access/qnetworkaccessmanager.cpp +++ b/src/network/access/qnetworkaccessmanager.cpp @@ -65,6 +65,8 @@ #include "QtNetwork/qsslconfiguration.h" #include "QtNetwork/qnetworkconfigmanager.h" +#include "qthread.h" + QT_BEGIN_NAMESPACE #ifndef QT_NO_HTTP @@ -1099,10 +1101,21 @@ void QNetworkAccessManagerPrivate::clearCache(QNetworkAccessManager *manager) { manager->d_func()->objectCache.clear(); manager->d_func()->authenticationManager->clearCache(); + + if (manager->d_func()->httpThread) { + // The thread will deleteLater() itself from its finished() signal + manager->d_func()->httpThread->quit(); + manager->d_func()->httpThread = 0; + } } QNetworkAccessManagerPrivate::~QNetworkAccessManagerPrivate() { + if (httpThread) { + // The thread will deleteLater() itself from its finished() signal + httpThread->quit(); + httpThread = 0; + } } #ifndef QT_NO_BEARERMANAGEMENT diff --git a/src/network/access/qnetworkaccessmanager.h b/src/network/access/qnetworkaccessmanager.h index 7ef009f..d67b8ac 100644 --- a/src/network/access/qnetworkaccessmanager.h +++ b/src/network/access/qnetworkaccessmanager.h @@ -156,6 +156,8 @@ protected: private: friend class QNetworkReplyImplPrivate; + friend class QNetworkAccessHttpBackend; + Q_DECLARE_PRIVATE(QNetworkAccessManager) Q_PRIVATE_SLOT(d_func(), void _q_replyFinished()) Q_PRIVATE_SLOT(d_func(), void _q_replySslErrors(QList)) diff --git a/src/network/access/qnetworkaccessmanager_p.h b/src/network/access/qnetworkaccessmanager_p.h index dba1fd2..ee6ad70 100644 --- a/src/network/access/qnetworkaccessmanager_p.h +++ b/src/network/access/qnetworkaccessmanager_p.h @@ -73,6 +73,7 @@ class QNetworkAccessManagerPrivate: public QObjectPrivate public: QNetworkAccessManagerPrivate() : networkCache(0), cookieJar(0), + httpThread(0), #ifndef QT_NO_NETWORKPROXY proxyFactory(0), #endif @@ -123,6 +124,8 @@ public: QNetworkCookieJar *cookieJar; + QThread *httpThread; + #ifndef QT_NO_NETWORKPROXY QNetworkProxy proxy; @@ -140,7 +143,7 @@ public: bool cookieJarCreated; // The cache with authorization data: - QNetworkAccessAuthenticationManager* authenticationManager; + QSharedPointer authenticationManager; // this cache can be used by individual backends to cache e.g. their TCP connections to a server // and use the connections for multiple requests. diff --git a/src/network/access/qnetworkreplyimpl.cpp b/src/network/access/qnetworkreplyimpl.cpp index 9069b14..39c4b07 100644 --- a/src/network/access/qnetworkreplyimpl.cpp +++ b/src/network/access/qnetworkreplyimpl.cpp @@ -57,7 +57,7 @@ Q_DECLARE_METATYPE(QSharedPointer) QT_BEGIN_NAMESPACE inline QNetworkReplyImplPrivate::QNetworkReplyImplPrivate() - : backend(0), outgoingData(0), outgoingDataBuffer(0), + : backend(0), outgoingData(0), copyDevice(0), cacheEnabled(false), cacheSaveDevice(0), notificationHandlingPaused(false), @@ -212,7 +212,7 @@ void QNetworkReplyImplPrivate::_q_bufferOutgoingData() if (!outgoingDataBuffer) { // first call, create our buffer - outgoingDataBuffer = new QRingBuffer(); + outgoingDataBuffer = QSharedPointer(new QRingBuffer()); QObject::connect(outgoingData, SIGNAL(readyRead()), q, SLOT(_q_bufferOutgoingData())); QObject::connect(outgoingData, SIGNAL(readChannelFinished()), q, SLOT(_q_bufferOutgoingDataFinished())); @@ -307,19 +307,21 @@ void QNetworkReplyImplPrivate::setup(QNetworkAccessManager::Operation op, const // in QtWebKit. QVariant synchronousHttpAttribute = req.attribute( static_cast(QNetworkRequest::DownloadBufferAttribute + 1)); - if (synchronousHttpAttribute.toBool()) { - backend->setSynchronous(true); - if (outgoingData && outgoingData->isSequential()) { - outgoingDataBuffer = new QRingBuffer(); - QByteArray data; - do { - data = outgoingData->readAll(); - if (data.isEmpty()) - break; - outgoingDataBuffer->append(data); - } while (1); - } + // The synchronous HTTP is a corner case, we will put all upload data in one big QByteArray in the outgoingDataBuffer. + // Yes, this is not the most efficient thing to do, but on the other hand synchronous XHR needs to die anyway. + if (synchronousHttpAttribute.toBool() && outgoingData) { + outgoingDataBuffer = QSharedPointer(new QRingBuffer()); + qint64 previousDataSize = 0; + do { + previousDataSize = outgoingDataBuffer->size(); + outgoingDataBuffer->append(outgoingData->readAll()); + } while (outgoingDataBuffer->size() != previousDataSize); } + + if (backend) + backend->setSynchronous(synchronousHttpAttribute.toBool()); + + if (outgoingData && backend && !backend->isSynchronous()) { // there is data to be uploaded, e.g. HTTP POST. @@ -349,10 +351,6 @@ void QNetworkReplyImplPrivate::setup(QNetworkAccessManager::Operation op, const } } } else { - // No outgoing data (e.g. HTTP GET request) - // or no backend - // if no backend, _q_startOperation will handle the error of this - // for HTTP, we want to send out the request as fast as possible to the network, without // invoking methods in a QueuedConnection #ifndef QT_NO_HTTP @@ -634,8 +632,9 @@ char* QNetworkReplyImplPrivate::getDownloadBuffer(qint64 size) { Q_Q(QNetworkReplyImpl); - // Check attribute() if allocating a buffer of that size can be allowed if (!downloadBuffer) { + // We are requested to create it + // Check attribute() if allocating a buffer of that size can be allowed QVariant bufferAllocationPolicy = request.attribute(QNetworkRequest::MaximumDownloadBufferSizeAttribute); if (bufferAllocationPolicy.isValid() && bufferAllocationPolicy.toLongLong() >= size) { downloadBufferCurrentSize = 0; @@ -650,6 +649,18 @@ char* QNetworkReplyImplPrivate::getDownloadBuffer(qint64 size) return downloadBuffer; } +void QNetworkReplyImplPrivate::setDownloadBuffer(QSharedPointer sp, qint64 size) +{ + Q_Q(QNetworkReplyImpl); + + downloadBufferPointer = sp; + downloadBuffer = downloadBufferPointer.data(); + downloadBufferCurrentSize = 0; + downloadBufferMaximumSize = size; + q->setAttribute(QNetworkRequest::DownloadBufferAttribute, qVariantFromValue > (downloadBufferPointer)); +} + + void QNetworkReplyImplPrivate::appendDownstreamDataDownloadBuffer(qint64 bytesReceived, qint64 bytesTotal) { Q_Q(QNetworkReplyImpl); @@ -746,6 +757,11 @@ void QNetworkReplyImplPrivate::finished() void QNetworkReplyImplPrivate::error(QNetworkReplyImpl::NetworkError code, const QString &errorMessage) { Q_Q(QNetworkReplyImpl); + // Can't set and emit multiple errors. + if (errorCode != QNetworkReply::NoError) { + qWarning() << "QNetworkReplyImplPrivate::error: Internal problem, this method must only be called once."; + return; + } errorCode = code; q->setErrorString(errorMessage); @@ -803,9 +819,6 @@ QNetworkReplyImpl::~QNetworkReplyImpl() // save had been properly finished. So if it is still enabled it means we got deleted/aborted. if (d->isCachingEnabled()) d->networkCache()->remove(url()); - - if (d->outgoingDataBuffer) - delete d->outgoingDataBuffer; } void QNetworkReplyImpl::abort() diff --git a/src/network/access/qnetworkreplyimpl_p.h b/src/network/access/qnetworkreplyimpl_p.h index 238bee5..1a9ab7e 100644 --- a/src/network/access/qnetworkreplyimpl_p.h +++ b/src/network/access/qnetworkreplyimpl_p.h @@ -164,6 +164,7 @@ public: void appendDownstreamData(QIODevice *data); void appendDownstreamData(const QByteArray &data); + void setDownloadBuffer(QSharedPointer sp, qint64 size); char* getDownloadBuffer(qint64 size); void appendDownstreamDataDownloadBuffer(qint64, qint64); @@ -175,7 +176,7 @@ public: QNetworkAccessBackend *backend; QIODevice *outgoingData; - QRingBuffer *outgoingDataBuffer; + QSharedPointer outgoingDataBuffer; QIODevice *copyDevice; QAbstractNetworkCache *networkCache() const; -- cgit v0.12