From 8ab072aff0527d3ef3e44cf1ceba7dca985a6f94 Mon Sep 17 00:00:00 2001 From: Markus Goetz Date: Wed, 15 Jul 2009 12:40:53 +0200 Subject: QNetworkAccessManager: HTTP download performance improvements Better usage of move semantics with implicit sharing to avoid detaching (=malloc/memcpy). Also some other improvements. Download performance improvement is around 20% according to the httpDownloadPerformance autotest. Reviewed-by: Thiago Macieira --- src/corelib/tools/qbytedata_p.h | 200 +++++++++++++++++++++ src/corelib/tools/tools.pri | 1 + src/network/access/qhttpnetworkconnection.cpp | 61 ++++--- src/network/access/qhttpnetworkconnection_p.h | 8 +- src/network/access/qhttpnetworkreply.cpp | 78 ++++---- src/network/access/qhttpnetworkreply_p.h | 12 +- src/network/access/qnetworkaccessbackend.cpp | 4 +- src/network/access/qnetworkaccessbackend_p.h | 2 +- src/network/access/qnetworkaccessdatabackend.cpp | 6 +- .../access/qnetworkaccessdebugpipebackend.cpp | 6 +- src/network/access/qnetworkaccessfilebackend.cpp | 6 +- src/network/access/qnetworkaccessftpbackend.cpp | 6 +- src/network/access/qnetworkaccesshttpbackend.cpp | 11 +- src/network/access/qnetworkreplyimpl.cpp | 40 +++-- src/network/access/qnetworkreplyimpl_p.h | 5 +- 15 files changed, 338 insertions(+), 108 deletions(-) create mode 100644 src/corelib/tools/qbytedata_p.h diff --git a/src/corelib/tools/qbytedata_p.h b/src/corelib/tools/qbytedata_p.h new file mode 100644 index 0000000..e8a4ddd --- /dev/null +++ b/src/corelib/tools/qbytedata_p.h @@ -0,0 +1,200 @@ +/**************************************************************************** +** +** Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies). +** Contact: Nokia Corporation (qt-info@nokia.com) +** +** This file is part of the QtCore module of the Qt Toolkit. +** +** $QT_BEGIN_LICENSE:LGPL$ +** No Commercial Usage +** This file contains pre-release code and may not be distributed. +** You may use this file in accordance with the terms and conditions +** contained in the either Technology Preview License Agreement or the +** Beta Release License Agreement. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 2.1 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 2.1 requirements +** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. +** +** In addition, as a special exception, Nokia gives you certain +** additional rights. These rights are described in the Nokia Qt LGPL +** Exception version 1.0, included in the file LGPL_EXCEPTION.txt in this +** package. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 3.0 as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL included in the +** packaging of this file. Please review the following information to +** ensure the GNU General Public License version 3.0 requirements will be +** met: http://www.gnu.org/copyleft/gpl.html. +** +** If you are unsure which license is appropriate for your use, please +** contact the sales department at http://www.qtsoftware.com/contact. +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#ifndef QBYTEDATA_H +#define QBYTEDATA_H + +#include + + +// this class handles a list of QByteArrays. It is a variant of QRingBuffer +// that avoid malloc/realloc/memcpy. +class QByteDataBuffer +{ +private: + QList buffers; + qint64 bufferCompleteSize; +public: + QByteDataBuffer() : bufferCompleteSize(0) + { + } + + ~QByteDataBuffer() + { + clear(); + } + + inline void append(QByteDataBuffer& other) + { + if (other.isEmpty()) + return; + + buffers.append(other.buffers); + bufferCompleteSize += other.byteAmount(); + } + + + inline void append(QByteArray& bd) + { + if (bd.isEmpty()) + return; + + buffers.append(bd); + bufferCompleteSize += bd.size(); + } + + inline void prepend(QByteArray& bd) + { + if (bd.isEmpty()) + return; + + buffers.prepend(bd); + bufferCompleteSize += bd.size(); + } + + // return the first QByteData. User of this function has to qFree() its .data! + // preferably use this function to read data. + inline QByteArray read() + { + bufferCompleteSize -= buffers.first().size(); + return buffers.takeFirst(); + } + + // return everything. User of this function has to qFree() its .data! + // avoid to use this, it might malloc and memcpy. + inline QByteArray readAll() + { + return read(byteAmount()); + } + + // return amount. User of this function has to qFree() its .data! + // avoid to use this, it might malloc and memcpy. + inline QByteArray read(qint64 amount) + { + amount = qMin(byteAmount(), amount); + QByteArray byteData; + byteData.resize(amount); + read(byteData.data(), byteData.size()); + return byteData; + } + + // return amount bytes. User of this function has to qFree() its .data! + // avoid to use this, it will memcpy. + qint64 read(char* dst, qint64 amount) + { + amount = qMin(amount, byteAmount()); + qint64 originalAmount = amount; + char *writeDst = dst; + + while (amount > 0) { + QByteArray first = buffers.takeFirst(); + if (amount >= first.size()) { + // take it completely + bufferCompleteSize -= first.size(); + amount -= first.size(); + memcpy(writeDst, first.constData(), first.size()); + writeDst += first.size(); + first.clear(); + } else { + // take a part of it & it is the last one to take + bufferCompleteSize -= amount; + memcpy(writeDst, first.constData(), amount); + + qint64 newFirstSize = first.size() - amount; + QByteArray newFirstData; + newFirstData.resize(newFirstSize); + memcpy(newFirstData.data(), first.constData() + amount, newFirstSize); + buffers.prepend(newFirstData); + + amount = 0; + first.clear(); + } + } + + return originalAmount; + } + + inline char getChar() + { + char c; + read(&c, 1); + return c; + } + + inline void clear() + { + buffers.clear(); + bufferCompleteSize = 0; + } + + // The byte count of all QByteArrays + inline qint64 byteAmount() const + { + return bufferCompleteSize; + } + + // the number of QByteArrays + inline qint64 bufferCount() const + { + return buffers.length(); + } + + inline bool isEmpty() const + { + return byteAmount() == 0; + } + + inline qint64 sizeNextBlock() const + { + if(buffers.isEmpty()) + return 0; + else + return buffers.first().size(); + } + + inline QByteArray& operator[](int i) + { + return buffers[i]; + } +}; + + +#endif // QBYTEDATA_H diff --git a/src/corelib/tools/tools.pri b/src/corelib/tools/tools.pri index c93a065..08c94ac 100644 --- a/src/corelib/tools/tools.pri +++ b/src/corelib/tools/tools.pri @@ -5,6 +5,7 @@ HEADERS += \ tools/qbitarray.h \ tools/qbytearray.h \ tools/qbytearraymatcher.h \ + tools/qbytedata_p.h \ tools/qcache.h \ tools/qchar.h \ tools/qcontainerfwd.h \ diff --git a/src/network/access/qhttpnetworkconnection.cpp b/src/network/access/qhttpnetworkconnection.cpp index f1da244..afcdf17 100644 --- a/src/network/access/qhttpnetworkconnection.cpp +++ b/src/network/access/qhttpnetworkconnection.cpp @@ -175,26 +175,43 @@ bool QHttpNetworkConnectionPrivate::isSocketReading(QAbstractSocket *socket) con return (i != -1 && (channels[i].state & ReadingState)); } +void QHttpNetworkConnectionPrivate::appendUncompressedData(QHttpNetworkReply &reply, QByteArray &qba) +{ + reply.d_func()->responseData.append(qba); + + // clear the original! helps with implicit sharing and + // avoiding memcpy when the user is reading the data + qba.clear(); +} -void QHttpNetworkConnectionPrivate::appendUncompressedData(QHttpNetworkReply &reply, const QByteArray &fragment) +void QHttpNetworkConnectionPrivate::appendUncompressedData(QHttpNetworkReply &reply, QByteDataBuffer &data) { - char *dst = reply.d_func()->responseData.reserve(fragment.size()); - qMemCopy(dst, fragment.constData(), fragment.size()); + reply.d_func()->responseData.append(data); + + // clear the original! helps with implicit sharing and + // avoiding memcpy when the user is reading the data + data.clear(); } -void QHttpNetworkConnectionPrivate::appendCompressedData(QHttpNetworkReply &reply, const QByteArray &fragment) +void QHttpNetworkConnectionPrivate::appendCompressedData(QHttpNetworkReply &reply, QByteDataBuffer &data) { - reply.d_func()->compressedData.append(fragment); + // Work in progress: Later we will directly use a list of QByteArray or a QRingBuffer + // instead of one QByteArray. + for(int i = 0; i < data.bufferCount(); i++) { + QByteArray &byteData = data[i]; + reply.d_func()->compressedData.append(byteData.constData(), byteData.size()); + } + data.clear(); } qint64 QHttpNetworkConnectionPrivate::uncompressedBytesAvailable(const QHttpNetworkReply &reply) const { - return reply.d_func()->responseData.size(); + return reply.d_func()->responseData.byteAmount(); } qint64 QHttpNetworkConnectionPrivate::uncompressedBytesAvailableNextBlock(const QHttpNetworkReply &reply) const { - return reply.d_func()->responseData.nextDataBlockSize(); + return reply.d_func()->responseData.sizeNextBlock(); } qint64 QHttpNetworkConnectionPrivate::compressedBytesAvailable(const QHttpNetworkReply &reply) const @@ -202,21 +219,6 @@ qint64 QHttpNetworkConnectionPrivate::compressedBytesAvailable(const QHttpNetwor return reply.d_func()->compressedData.size(); } -qint64 QHttpNetworkConnectionPrivate::read(QHttpNetworkReply &reply, QByteArray &data, qint64 maxSize) -{ - QRingBuffer *rb = &reply.d_func()->responseData; - if (maxSize == -1 || maxSize >= rb->size()) { - // read the whole data - data = rb->readAll(); - rb->clear(); - } else { - // read only the requested length - data.resize(maxSize); - rb->read(data.data(), maxSize); - } - return data.size(); -} - void QHttpNetworkConnectionPrivate::eraseData(QHttpNetworkReply *reply) { reply->d_func()->compressedData.clear(); @@ -556,6 +558,8 @@ bool QHttpNetworkConnectionPrivate::expand(QAbstractSocket *socket, QHttpNetwork reply->d_func()->totalProgress += inflated.size(); appendUncompressedData(*reply, inflated); if (shouldEmitSignals(reply)) { + // important: At the point of this readyRead(), inflated must be cleared, + // else implicit sharing will trigger memcpy when the user is reading data! emit reply->readyRead(); // make sure that the reply is valid if (channels[i].reply != reply) @@ -672,18 +676,19 @@ void QHttpNetworkConnectionPrivate::receiveReply(QAbstractSocket *socket, QHttpN { // use the traditional slower reading (for compressed encoding, chunked encoding, // no content-length etc) - QBuffer fragment; - fragment.open(QIODevice::WriteOnly); - bytes = reply->d_func()->readBody(socket, &fragment); + QByteDataBuffer byteDatas; + bytes = reply->d_func()->readBody(socket, &byteDatas); if (bytes) { if (reply->d_func()->autoDecompress) - appendCompressedData(*reply, fragment.data()); + appendCompressedData(*reply, byteDatas); else - appendUncompressedData(*reply, fragment.data()); + appendUncompressedData(*reply, byteDatas); if (!reply->d_func()->autoDecompress) { - reply->d_func()->totalProgress += fragment.size(); + reply->d_func()->totalProgress += bytes; if (shouldEmitSignals(reply)) { + // important: At the point of this readyRead(), the byteDatas list must be empty, + // else implicit sharing will trigger memcpy when the user is reading data! emit reply->readyRead(); // make sure that the reply is valid if (channels[i].reply != reply) diff --git a/src/network/access/qhttpnetworkconnection_p.h b/src/network/access/qhttpnetworkconnection_p.h index a0813d4..842a2f4 100644 --- a/src/network/access/qhttpnetworkconnection_p.h +++ b/src/network/access/qhttpnetworkconnection_p.h @@ -79,6 +79,7 @@ QT_BEGIN_NAMESPACE class QHttpNetworkRequest; class QHttpNetworkReply; +class QByteArray; class QHttpNetworkConnectionPrivate; class Q_AUTOTEST_EXPORT QHttpNetworkConnection : public QObject @@ -255,15 +256,14 @@ public: bool pendingAuthSignal; // there is an incomplete authentication signal bool pendingProxyAuthSignal; // there is an incomplete proxy authentication signal - void appendUncompressedData(QHttpNetworkReply &reply, const QByteArray &fragment); - void appendCompressedData(QHttpNetworkReply &reply, const QByteArray &fragment); + void appendUncompressedData(QHttpNetworkReply &reply, QByteArray &qba); + void appendUncompressedData(QHttpNetworkReply &reply, QByteDataBuffer &data); + void appendCompressedData(QHttpNetworkReply &reply, QByteDataBuffer &data); qint64 uncompressedBytesAvailable(const QHttpNetworkReply &reply) const; qint64 uncompressedBytesAvailableNextBlock(const QHttpNetworkReply &reply) const; qint64 compressedBytesAvailable(const QHttpNetworkReply &reply) const; - qint64 read(QHttpNetworkReply &reply, QByteArray &data, qint64 maxSize); - void emitReplyError(QAbstractSocket *socket, QHttpNetworkReply *reply, QNetworkReply::NetworkError errorCode); bool handleAuthenticateChallenge(QAbstractSocket *socket, QHttpNetworkReply *reply, bool isProxy, bool &resend); void allDone(QAbstractSocket *socket, QHttpNetworkReply *reply); diff --git a/src/network/access/qhttpnetworkreply.cpp b/src/network/access/qhttpnetworkreply.cpp index 7a616aa..2fe0d78 100644 --- a/src/network/access/qhttpnetworkreply.cpp +++ b/src/network/access/qhttpnetworkreply.cpp @@ -176,15 +176,6 @@ qint64 QHttpNetworkReply::bytesAvailableNextBlock() const return -1; } -QByteArray QHttpNetworkReply::read(qint64 maxSize) -{ - Q_D(QHttpNetworkReply); - QByteArray data; - if (d->connection) - d->connection->d_func()->read(*this, data, maxSize); - return data; -} - QByteArray QHttpNetworkReply::readAny() { Q_D(QHttpNetworkReply); @@ -203,7 +194,7 @@ QHttpNetworkReplyPrivate::QHttpNetworkReplyPrivate(const QUrl &newUrl) majorVersion(0), minorVersion(0), bodyLength(0), contentRead(0), totalProgress(0), chunkedTransferEncoding(0), currentChunkSize(0), currentChunkRead(0), connection(0), initInflate(false), - autoDecompress(false), responseData(0), requestIsPrepared(false) + autoDecompress(false), responseData(), requestIsPrepared(false) { } @@ -561,17 +552,19 @@ bool QHttpNetworkReplyPrivate::connectionCloseEnabled() // note this function can only be used for non-chunked, non-compressed with // known content length -qint64 QHttpNetworkReplyPrivate::readBodyFast(QAbstractSocket *socket, QRingBuffer *rb) -{ - quint64 toBeRead = qMin(socket->bytesAvailable(), bodyLength - contentRead); - char* dst = rb->reserve(toBeRead); - qint64 haveRead = socket->read(dst, toBeRead); +qint64 QHttpNetworkReplyPrivate::readBodyFast(QAbstractSocket *socket, QByteDataBuffer *rb) +{ + qint64 toBeRead = qMin(socket->bytesAvailable(), bodyLength - contentRead); + QByteArray bd; + bd.resize(toBeRead); + qint64 haveRead = socket->read(bd.data(), bd.size()); if (haveRead == -1) { - rb->chop(toBeRead); + bd.clear(); return 0; // ### error checking here; } + bd.resize(haveRead); - rb->chop(toBeRead - haveRead); + rb->append(bd); if (contentRead + haveRead == bodyLength) { state = AllDoneState; @@ -583,7 +576,7 @@ qint64 QHttpNetworkReplyPrivate::readBodyFast(QAbstractSocket *socket, QRingBuff } -qint64 QHttpNetworkReplyPrivate::readBody(QAbstractSocket *socket, QIODevice *out) +qint64 QHttpNetworkReplyPrivate::readBody(QAbstractSocket *socket, QByteDataBuffer *out) { qint64 bytes = 0; if (isChunked()) { @@ -601,33 +594,35 @@ qint64 QHttpNetworkReplyPrivate::readBody(QAbstractSocket *socket, QIODevice *ou return bytes; } -qint64 QHttpNetworkReplyPrivate::readReplyBodyRaw(QIODevice *in, QIODevice *out, qint64 size) +qint64 QHttpNetworkReplyPrivate::readReplyBodyRaw(QIODevice *in, QByteDataBuffer *out, qint64 size) { qint64 bytes = 0; Q_ASSERT(in); Q_ASSERT(out); int toBeRead = qMin(128*1024, qMin(size, in->bytesAvailable())); - QByteArray raw(toBeRead, 0); - while (size > 0) { - qint64 read = in->read(raw.data(), raw.size()); - if (read == 0) - return bytes; - // ### error checking here - qint64 written = out->write(raw.data(), read); - if (written == 0) + while (toBeRead > 0) { + QByteArray byteData; + byteData.resize(toBeRead); + qint64 haveRead = in->read(byteData.data(), byteData.size()); + if (haveRead <= 0) { + // ### error checking here + byteData.clear(); return bytes; - if (read != written) - qDebug() << "### read" << read << "written" << written; - bytes += read; - size -= read; - out->waitForBytesWritten(-1); // throttle + } + + byteData.resize(haveRead); + out->append(byteData); + bytes += haveRead; + size -= haveRead; + + toBeRead = qMin(128*1024, qMin(size, in->bytesAvailable())); } return bytes; } -qint64 QHttpNetworkReplyPrivate::readReplyBodyChunked(QIODevice *in, QIODevice *out) +qint64 QHttpNetworkReplyPrivate::readReplyBodyChunked(QIODevice *in, QByteDataBuffer *out) { qint64 bytes = 0; while (in->bytesAvailable()) { // while we can read from input @@ -648,17 +643,14 @@ qint64 QHttpNetworkReplyPrivate::readReplyBodyChunked(QIODevice *in, QIODevice * state = AllDoneState; break; } - // otherwise, read data - qint64 readSize = qMin(in->bytesAvailable(), currentChunkSize - currentChunkRead); - QByteArray buffer(readSize, 0); - qint64 read = in->read(buffer.data(), readSize); - bytes += read; - currentChunkRead += read; - qint64 written = out->write(buffer); - Q_UNUSED(written); // Avoid compile warning when building release - Q_ASSERT(read == written); + + // otherwise, try to read what is missing for this chunk + qint64 haveRead = readReplyBodyRaw (in, out, currentChunkSize - currentChunkRead); + currentChunkRead += haveRead; + bytes += haveRead; + // ### error checking here - out->waitForBytesWritten(-1); + } return bytes; } diff --git a/src/network/access/qhttpnetworkreply_p.h b/src/network/access/qhttpnetworkreply_p.h index 5eb70ce..fbbee12 100644 --- a/src/network/access/qhttpnetworkreply_p.h +++ b/src/network/access/qhttpnetworkreply_p.h @@ -80,6 +80,7 @@ static const unsigned char gz_magic[2] = {0x1f, 0x8b}; // gzip magic header #include #include #include +#include QT_BEGIN_NAMESPACE @@ -122,7 +123,6 @@ public: qint64 bytesAvailable() const; qint64 bytesAvailableNextBlock() const; - QByteArray read(qint64 maxSize = -1); QByteArray readAny(); bool isFinished() const; @@ -160,14 +160,14 @@ public: bool parseStatus(const QByteArray &status); qint64 readHeader(QAbstractSocket *socket); void parseHeader(const QByteArray &header); - qint64 readBody(QAbstractSocket *socket, QIODevice *out); - qint64 readBodyFast(QAbstractSocket *socket, QRingBuffer *rb); + qint64 readBody(QAbstractSocket *socket, QByteDataBuffer *out); + qint64 readBodyFast(QAbstractSocket *socket, QByteDataBuffer *rb); bool findChallenge(bool forProxy, QByteArray &challenge) const; QAuthenticatorPrivate::Method authenticationMethod(bool isProxy) const; void clear(); - qint64 readReplyBodyRaw(QIODevice *in, QIODevice *out, qint64 size); - qint64 readReplyBodyChunked(QIODevice *in, QIODevice *out); + qint64 readReplyBodyRaw(QIODevice *in, QByteDataBuffer *out, qint64 size); + qint64 readReplyBodyChunked(QIODevice *in, QByteDataBuffer *out); qint64 getChunkSize(QIODevice *in, qint64 *chunkSize); qint64 bytesAvailable() const; @@ -209,7 +209,7 @@ public: #endif bool autoDecompress; - QRingBuffer responseData; // uncompressed body + QByteDataBuffer responseData; // uncompressed body QByteArray compressedData; // compressed body (temporary) bool requestIsPrepared; }; diff --git a/src/network/access/qnetworkaccessbackend.cpp b/src/network/access/qnetworkaccessbackend.cpp index 88ae894..9e17b54 100644 --- a/src/network/access/qnetworkaccessbackend.cpp +++ b/src/network/access/qnetworkaccessbackend.cpp @@ -217,9 +217,9 @@ qint64 QNetworkAccessBackend::nextDownstreamBlockSize() const return reply->nextDownstreamBlockSize(); } -void QNetworkAccessBackend::writeDownstreamData(const QByteArray &data) +void QNetworkAccessBackend::writeDownstreamData(QByteDataBuffer &list) { - reply->appendDownstreamData(data); + reply->appendDownstreamData(list); } void QNetworkAccessBackend::writeDownstreamData(QIODevice *data) diff --git a/src/network/access/qnetworkaccessbackend_p.h b/src/network/access/qnetworkaccessbackend_p.h index 21cb4a6..553b795 100644 --- a/src/network/access/qnetworkaccessbackend_p.h +++ b/src/network/access/qnetworkaccessbackend_p.h @@ -166,7 +166,7 @@ protected: // these functions control the downstream mechanism // that is, data that has come via the connection and is going out the backend qint64 nextDownstreamBlockSize() const; - void writeDownstreamData(const QByteArray &data); + void writeDownstreamData(QByteDataBuffer &list); public slots: // for task 251801, needs to be a slot to be called asynchronously diff --git a/src/network/access/qnetworkaccessdatabackend.cpp b/src/network/access/qnetworkaccessdatabackend.cpp index 609f0c5..4436cf4 100644 --- a/src/network/access/qnetworkaccessdatabackend.cpp +++ b/src/network/access/qnetworkaccessdatabackend.cpp @@ -117,7 +117,11 @@ void QNetworkAccessDataBackend::open() setHeader(QNetworkRequest::ContentLengthHeader, payload.size()); emit metaDataChanged(); - writeDownstreamData(payload); + QByteDataBuffer list; + list.append(payload); + payload.clear(); // important because of implicit sharing! + writeDownstreamData(list); + finished(); return; } diff --git a/src/network/access/qnetworkaccessdebugpipebackend.cpp b/src/network/access/qnetworkaccessdebugpipebackend.cpp index 54fcddd..2b3c128 100644 --- a/src/network/access/qnetworkaccessdebugpipebackend.cpp +++ b/src/network/access/qnetworkaccessdebugpipebackend.cpp @@ -155,7 +155,11 @@ void QNetworkAccessDebugPipeBackend::pushFromSocketToDownstream() // have read something buffer.resize(haveRead); bytesDownloaded += haveRead; - writeDownstreamData(buffer); + + QByteDataBuffer list; + list.append(buffer); + buffer.clear(); // important because of implicit sharing! + writeDownstreamData(list); } } } diff --git a/src/network/access/qnetworkaccessfilebackend.cpp b/src/network/access/qnetworkaccessfilebackend.cpp index e3fc8bf..533fc75 100644 --- a/src/network/access/qnetworkaccessfilebackend.cpp +++ b/src/network/access/qnetworkaccessfilebackend.cpp @@ -263,7 +263,11 @@ bool QNetworkAccessFileBackend::readMoreFromFile() data.resize(actuallyRead); totalBytes += actuallyRead; - writeDownstreamData(data); + + QByteDataBuffer list; + list.append(data); + data.clear(); // important because of implicit sharing! + writeDownstreamData(list); } return true; } diff --git a/src/network/access/qnetworkaccessftpbackend.cpp b/src/network/access/qnetworkaccessftpbackend.cpp index d6276a3..911b31a 100644 --- a/src/network/access/qnetworkaccessftpbackend.cpp +++ b/src/network/access/qnetworkaccessftpbackend.cpp @@ -355,7 +355,11 @@ void QNetworkAccessFtpBackend::ftpDone() void QNetworkAccessFtpBackend::ftpReadyRead() { - writeDownstreamData(ftp->readAll()); + QByteArray data = ftp->readAll(); + QByteDataBuffer list; + list.append(data); + data.clear(); // important because of implicit sharing! + writeDownstreamData(list); } void QNetworkAccessFtpBackend::ftpRawCommandReply(int code, const QString &text) diff --git a/src/network/access/qnetworkaccesshttpbackend.cpp b/src/network/access/qnetworkaccesshttpbackend.cpp index db84e58..9c36026 100644 --- a/src/network/access/qnetworkaccesshttpbackend.cpp +++ b/src/network/access/qnetworkaccesshttpbackend.cpp @@ -653,10 +653,15 @@ void QNetworkAccessHttpBackend::readFromHttp() // this is not a critical thing since it is already in the // memory anyway - while (httpReply->bytesAvailable() != 0 && nextDownstreamBlockSize() != 0) { - const QByteArray data = httpReply->readAny(); - writeDownstreamData(data); + QByteDataBuffer list; + + while (httpReply->bytesAvailable() != 0 && nextDownstreamBlockSize() != 0 && nextDownstreamBlockSize() > list.byteAmount()) { + QByteArray data = httpReply->readAny(); + list.append(data); } + + if (!list.isEmpty()) + writeDownstreamData(list); } void QNetworkAccessHttpBackend::replyFinished() diff --git a/src/network/access/qnetworkreplyimpl.cpp b/src/network/access/qnetworkreplyimpl.cpp index 55b8b7f..44ae328 100644 --- a/src/network/access/qnetworkreplyimpl.cpp +++ b/src/network/access/qnetworkreplyimpl.cpp @@ -103,16 +103,17 @@ void QNetworkReplyImplPrivate::_q_copyReadyRead() break; bytesToRead = qBound(1, bytesToRead, copyDevice->bytesAvailable()); - char *ptr = readBuffer.reserve(bytesToRead); - qint64 bytesActuallyRead = copyDevice->read(ptr, bytesToRead); + QByteArray byteData; + byteData.resize(bytesToRead); + qint64 bytesActuallyRead = copyDevice->read(byteData.data(), byteData.size()); if (bytesActuallyRead == -1) { - readBuffer.chop(bytesToRead); + byteData.clear(); backendNotify(NotifyCopyFinished); break; } - if (bytesActuallyRead != bytesToRead) - readBuffer.chop(bytesToRead - bytesActuallyRead); + byteData.resize(bytesActuallyRead); + readBuffer.append(byteData); if (!copyDevice->isSequential() && copyDevice->atEnd()) { backendNotify(NotifyCopyFinished); @@ -384,19 +385,17 @@ qint64 QNetworkReplyImplPrivate::nextDownstreamBlockSize() const if (readBufferMaxSize == 0) return DesiredBufferSize; - return qMax(0, readBufferMaxSize - readBuffer.size()); + return qMax(0, readBufferMaxSize - readBuffer.byteAmount()); } // we received downstream data and send this to the cache // and to our readBuffer (which in turn gets read by the user of QNetworkReply) -void QNetworkReplyImplPrivate::appendDownstreamData(const QByteArray &data) +void QNetworkReplyImplPrivate::appendDownstreamData(QByteDataBuffer &data) { Q_Q(QNetworkReplyImpl); if (!q->isOpen()) return; - readBuffer.append(data); - if (cacheEnabled && !cacheSaveDevice) { // save the meta data QNetworkCacheMetaData metaData; @@ -415,10 +414,19 @@ void QNetworkReplyImplPrivate::appendDownstreamData(const QByteArray &data) } } - if (cacheSaveDevice) - cacheSaveDevice->write(data); + qint64 bytesWritten = 0; + for (int i = 0; i < data.bufferCount(); i++) { + QByteArray item = data[i]; + + if (cacheSaveDevice) + cacheSaveDevice->write(item.constData(), item.size()); + readBuffer.append(item); + + bytesWritten += item.size(); + } + data.clear(); - bytesDownloaded += data.size(); + bytesDownloaded += bytesWritten; lastBytesDownloaded = bytesDownloaded; QPointer qq = q; @@ -427,6 +435,8 @@ void QNetworkReplyImplPrivate::appendDownstreamData(const QByteArray &data) pauseNotificationHandling(); emit q->downloadProgress(bytesDownloaded, totalSize.isNull() ? Q_INT64_C(-1) : totalSize.toLongLong()); + // important: At the point of this readyRead(), the data parameter list must be empty, + // else implicit sharing will trigger memcpy when the user is reading data! emit q->readyRead(); // hopefully we haven't been deleted here @@ -602,14 +612,14 @@ void QNetworkReplyImpl::close() */ qint64 QNetworkReplyImpl::bytesAvailable() const { - return QNetworkReply::bytesAvailable() + d_func()->readBuffer.size(); + return QNetworkReply::bytesAvailable() + d_func()->readBuffer.byteAmount(); } void QNetworkReplyImpl::setReadBufferSize(qint64 size) { Q_D(QNetworkReplyImpl); if (size > d->readBufferMaxSize && - size == d->readBuffer.size()) + size > d->readBuffer.byteAmount()) d->backendNotify(QNetworkReplyImplPrivate::NotifyDownstreamReadyWrite); QNetworkReply::setReadBufferSize(size); @@ -657,7 +667,7 @@ qint64 QNetworkReplyImpl::readData(char *data, qint64 maxlen) return 1; } - maxlen = qMin(maxlen, d->readBuffer.size()); + maxlen = qMin(maxlen, d->readBuffer.byteAmount()); return d->readBuffer.read(data, maxlen); } diff --git a/src/network/access/qnetworkreplyimpl_p.h b/src/network/access/qnetworkreplyimpl_p.h index 454185a..83a8aca 100644 --- a/src/network/access/qnetworkreplyimpl_p.h +++ b/src/network/access/qnetworkreplyimpl_p.h @@ -61,6 +61,7 @@ #include "QtCore/qqueue.h" #include "QtCore/qbuffer.h" #include "private/qringbuffer_p.h" +#include "private/qbytedata_p.h" QT_BEGIN_NAMESPACE @@ -144,7 +145,7 @@ public: void consume(qint64 count); void emitUploadProgress(qint64 bytesSent, qint64 bytesTotal); qint64 nextDownstreamBlockSize() const; - void appendDownstreamData(const QByteArray &data); + void appendDownstreamData(QByteDataBuffer &data); void appendDownstreamData(QIODevice *data); void finished(); void error(QNetworkReply::NetworkError code, const QString &errorString); @@ -172,7 +173,7 @@ public: QList proxyList; #endif - QRingBuffer readBuffer; + QByteDataBuffer readBuffer; qint64 bytesDownloaded; qint64 lastBytesDownloaded; qint64 bytesUploaded; -- cgit v0.12