From 26af575213763c8202f1a142f42257d473076529 Mon Sep 17 00:00:00 2001 From: Markus Goetz Date: Thu, 12 Aug 2010 11:20:14 +0200 Subject: QNAM Zerocopy: QNAM implementation part Reviewed-by: Peter Hartmann Reviewed-by: Jeremy Katz Reviewed-by: Aleksandar Sasha Babic --- src/network/access/qnetworkaccessbackend.cpp | 11 ++++ src/network/access/qnetworkaccessbackend_p.h | 4 ++ src/network/access/qnetworkaccesshttpbackend.cpp | 33 +++++++++- src/network/access/qnetworkaccesshttpbackend_p.h | 1 + src/network/access/qnetworkreplyimpl.cpp | 80 ++++++++++++++++++++++++ src/network/access/qnetworkreplyimpl_p.h | 10 +++ 6 files changed, 138 insertions(+), 1 deletion(-) diff --git a/src/network/access/qnetworkaccessbackend.cpp b/src/network/access/qnetworkaccessbackend.cpp index 60f7dc6..45495f7 100644 --- a/src/network/access/qnetworkaccessbackend.cpp +++ b/src/network/access/qnetworkaccessbackend.cpp @@ -252,6 +252,17 @@ void QNetworkAccessBackend::writeDownstreamData(QIODevice *data) reply->appendDownstreamData(data); } +// not actually appending data, it was already written to the user buffer +void QNetworkAccessBackend::writeDownstreamDataDownloadBuffer(qint64 bytesReceived, qint64 bytesTotal) +{ + reply->appendDownstreamDataDownloadBuffer(bytesReceived, bytesTotal); +} + +char* QNetworkAccessBackend::getDownloadBuffer(qint64 size) +{ + return reply->getDownloadBuffer(size); +} + QVariant QNetworkAccessBackend::header(QNetworkRequest::KnownHeaders header) const { return reply->q_func()->header(header); diff --git a/src/network/access/qnetworkaccessbackend_p.h b/src/network/access/qnetworkaccessbackend_p.h index 4fe6de6..9f8a01f 100644 --- a/src/network/access/qnetworkaccessbackend_p.h +++ b/src/network/access/qnetworkaccessbackend_p.h @@ -177,6 +177,10 @@ protected: qint64 nextDownstreamBlockSize() const; void writeDownstreamData(QByteDataBuffer &list); + // not actually appending data, it was already written to the user buffer + void writeDownstreamDataDownloadBuffer(qint64, qint64); + char* getDownloadBuffer(qint64); + public slots: // for task 251801, needs to be a slot to be called asynchronously void writeDownstreamData(QIODevice *data); diff --git a/src/network/access/qnetworkaccesshttpbackend.cpp b/src/network/access/qnetworkaccesshttpbackend.cpp index f617244..8b9a99f 100644 --- a/src/network/access/qnetworkaccesshttpbackend.cpp +++ b/src/network/access/qnetworkaccesshttpbackend.cpp @@ -542,6 +542,9 @@ void QNetworkAccessHttpBackend::postRequest() break; // can't happen } + bool encrypt = (url().scheme().toLower() == QLatin1String("https")); + httpRequest.setSsl(encrypt); + httpRequest.setUrl(url()); QList headers = request().rawHeaderList(); @@ -595,7 +598,6 @@ void QNetworkAccessHttpBackend::postRequest() httpReply->ignoreSslErrors(pendingIgnoreSslErrorsList); #endif - connect(httpReply, SIGNAL(readyRead()), SLOT(replyReadyRead())); connect(httpReply, SIGNAL(finished()), SLOT(replyFinished())); connect(httpReply, SIGNAL(finishedWithError(QNetworkReply::NetworkError,QString)), SLOT(httpError(QNetworkReply::NetworkError,QString))); @@ -859,9 +861,33 @@ void QNetworkAccessHttpBackend::replyHeaderChanged() if (!isCachingEnabled()) setCachingEnabled(true); } + + // Check if a download buffer is supported from the HTTP reply + char *buf = 0; + if (httpReply->supportsUserProvidedDownloadBuffer()) { + // Check if a download buffer is supported by the user + buf = getDownloadBuffer(httpReply->contentLength()); + if (buf) { + httpReply->setUserProvidedDownloadBuffer(buf); + // If there is a download buffer we react on the progress signal + connect(httpReply, SIGNAL(dataReadProgress(int,int)), SLOT(replyDownloadProgressSlot(int,int))); + } + } + + // If there is no buffer, we react on the readyRead signal + if (!buf) { + connect(httpReply, SIGNAL(readyRead()), SLOT(replyReadyRead())); + } + metaDataChanged(); } +void QNetworkAccessHttpBackend::replyDownloadProgressSlot(int received, int total) +{ + // we can be sure here that there is a download buffer + writeDownstreamDataDownloadBuffer(received, total); +} + void QNetworkAccessHttpBackend::httpAuthenticationRequired(const QHttpNetworkRequest &, QAuthenticator *auth) { @@ -1169,6 +1195,11 @@ bool QNetworkAccessHttpBackend::canResume() const return false; } + // If we're using a download buffer then we don't support resuming/migration + // right now. Too much trouble. + if (httpReply->userProvidedDownloadBuffer()) + return false; + return true; } diff --git a/src/network/access/qnetworkaccesshttpbackend_p.h b/src/network/access/qnetworkaccesshttpbackend_p.h index c4c88ae..fb12781 100644 --- a/src/network/access/qnetworkaccesshttpbackend_p.h +++ b/src/network/access/qnetworkaccesshttpbackend_p.h @@ -104,6 +104,7 @@ private slots: void replyReadyRead(); void replyFinished(); void replyHeaderChanged(); + void replyDownloadProgressSlot(int,int); void httpAuthenticationRequired(const QHttpNetworkRequest &request, QAuthenticator *auth); void httpCacheCredentials(const QHttpNetworkRequest &request, QAuthenticator *auth); void httpError(QNetworkReply::NetworkError error, const QString &errorString); diff --git a/src/network/access/qnetworkreplyimpl.cpp b/src/network/access/qnetworkreplyimpl.cpp index 3798ac2..c8808af 100644 --- a/src/network/access/qnetworkreplyimpl.cpp +++ b/src/network/access/qnetworkreplyimpl.cpp @@ -49,11 +49,15 @@ #include "QtNetwork/qnetworksession.h" #include "qnetworkaccesshttpbackend_p.h" #include "qnetworkaccessmanager_p.h" +#include #include QT_BEGIN_NAMESPACE +typedef QSharedPointer > QVarLengthArraySharedPointer; +Q_DECLARE_METATYPE(QVarLengthArraySharedPointer) + inline QNetworkReplyImplPrivate::QNetworkReplyImplPrivate() : backend(0), outgoingData(0), outgoingDataBuffer(0), copyDevice(0), @@ -62,6 +66,8 @@ inline QNetworkReplyImplPrivate::QNetworkReplyImplPrivate() bytesDownloaded(0), lastBytesDownloaded(-1), bytesUploaded(-1), preMigrationDownloaded(-1), httpStatusCode(0), state(Idle) + , downloadBuffer(0) + , downloadBufferPosition(0) { } @@ -125,6 +131,10 @@ void QNetworkReplyImplPrivate::_q_copyReadyRead() if (!copyDevice || !q->isOpen()) return; + // FIXME Optimize to use download buffer if it is a QBuffer. + // Needs to be done where sendCacheContents() (?) of HTTP is emitting + // metaDataChanged ? + forever { qint64 bytesToRead = nextDownstreamBlockSize(); if (bytesToRead == 0) @@ -590,6 +600,55 @@ void QNetworkReplyImplPrivate::appendDownstreamData(const QByteArray &data) qFatal("QNetworkReplyImplPrivate::appendDownstreamData not implemented"); } +char* QNetworkReplyImplPrivate::getDownloadBuffer(qint64 size) +{ + Q_Q(QNetworkReplyImpl); + + // Check attribute() if allocating a buffer of that size can be allowed + if (!downloadBuffer) { + QVariant bufferAllocationPolicy = request.attribute(QNetworkRequest::MaximumDownloadBufferSizeAttribute); + if (bufferAllocationPolicy.isValid() && bufferAllocationPolicy.toLongLong() >= size) { + downloadBufferArray = QSharedPointer >(new QVarLengthArray()); + downloadBufferArray->reserve(size); + + downloadBuffer = downloadBufferArray->data(); + + q->setAttribute(QNetworkRequest::DownloadBufferAttribute, qVariantFromValue > > (downloadBufferArray)); + } + } + + return downloadBuffer; +} + +void QNetworkReplyImplPrivate::appendDownstreamDataDownloadBuffer(qint64 bytesReceived, qint64 bytesTotal) +{ + Q_Q(QNetworkReplyImpl); + if (!q->isOpen()) + return; + + if (cacheEnabled && !cacheSaveDevice) + initCacheSaveDevice(); + + if (cacheSaveDevice && bytesReceived == bytesTotal) { +// if (lastBytesDownloaded == -1) +// lastBytesDownloaded = 0; +// cacheSaveDevice->write(downloadBuffer + lastBytesDownloaded, bytesReceived - lastBytesDownloaded); + + // Write everything in one go if we use a download buffer. might be more performant. + cacheSaveDevice->write(downloadBuffer, bytesTotal); + } + + bytesDownloaded = bytesReceived; + lastBytesDownloaded = bytesReceived; + + // Update the array so our user (e.g. QtWebKit) knows the real size + if (bytesReceived > 0) + downloadBufferArray->resize(bytesReceived); + + emit q->downloadProgress(bytesDownloaded, bytesTotal); + emit q->readyRead(); +} + void QNetworkReplyImplPrivate::finished() { Q_Q(QNetworkReplyImpl); @@ -784,6 +843,13 @@ bool QNetworkReplyImpl::canReadLine () const */ qint64 QNetworkReplyImpl::bytesAvailable() const { + // Special case for the "zero copy" download buffer + Q_D(const QNetworkReplyImpl); + if (d->downloadBuffer) { + qint64 maxAvail = d->downloadBufferArray->size() - d->downloadBufferPosition; + return QNetworkReply::bytesAvailable() + maxAvail; + } + return QNetworkReply::bytesAvailable() + d_func()->readBuffer.byteAmount(); } @@ -838,8 +904,22 @@ void QNetworkReplyImpl::ignoreSslErrorsImplementation(const QList &er qint64 QNetworkReplyImpl::readData(char *data, qint64 maxlen) { Q_D(QNetworkReplyImpl); + + // Special case code if we have the "zero copy" download buffer + if (d->downloadBuffer) { + qint64 maxAvail = qMin(d->downloadBufferArray->size() - d->downloadBufferPosition, maxlen); + if (maxAvail == 0) + return d->state == QNetworkReplyImplPrivate::Finished ? -1 : 0; + // FIXME what about "Aborted" state? + qMemCopy(data, d->downloadBuffer + d->downloadBufferPosition, maxAvail); + d->downloadBufferPosition += maxAvail; + return maxAvail; + } + + if (d->readBuffer.isEmpty()) return d->state == QNetworkReplyImplPrivate::Finished ? -1 : 0; + // FIXME what about "Aborted" state? d->backendNotify(QNetworkReplyImplPrivate::NotifyDownstreamReadyWrite); if (maxlen == 1) { diff --git a/src/network/access/qnetworkreplyimpl_p.h b/src/network/access/qnetworkreplyimpl_p.h index 38084bd..ab11ebe 100644 --- a/src/network/access/qnetworkreplyimpl_p.h +++ b/src/network/access/qnetworkreplyimpl_p.h @@ -62,6 +62,7 @@ #include "QtCore/qbuffer.h" #include "private/qringbuffer_p.h" #include "private/qbytedata_p.h" +#include QT_BEGIN_NAMESPACE @@ -163,6 +164,9 @@ public: void appendDownstreamData(QIODevice *data); void appendDownstreamData(const QByteArray &data); + char* getDownloadBuffer(qint64 size); + void appendDownstreamDataDownloadBuffer(qint64, qint64); + void finished(); void error(QNetworkReply::NetworkError code, const QString &errorString); void metaDataChanged(); @@ -191,6 +195,7 @@ public: QList proxyList; #endif + // Used for normal downloading. For "zero copy" the downloadBuffer is used QByteDataBuffer readBuffer; qint64 bytesDownloaded; qint64 lastBytesDownloaded; @@ -202,6 +207,11 @@ public: State state; + // only used when the "zero copy" style is used. Else readBuffer is used. + QSharedPointer< QVarLengthArray > downloadBufferArray; + char* downloadBuffer; + qint64 downloadBufferPosition; + Q_DECLARE_PUBLIC(QNetworkReplyImpl) }; -- cgit v0.12