diff options
Diffstat (limited to 'src/network/access')
23 files changed, 596 insertions, 651 deletions
diff --git a/src/network/access/qhttpnetworkconnection.cpp b/src/network/access/qhttpnetworkconnection.cpp index 5940fba..d4bf008 100644 --- a/src/network/access/qhttpnetworkconnection.cpp +++ b/src/network/access/qhttpnetworkconnection.cpp @@ -40,6 +40,7 @@ ****************************************************************************/ #include "qhttpnetworkconnection_p.h" +#include "private/qnoncontiguousbytedevice_p.h" #include <private/qnetworkrequest_p.h> #include <private/qobject_p.h> #include <private/qauthenticator_p.h> @@ -71,6 +72,7 @@ QHttpNetworkConnectionPrivate::QHttpNetworkConnectionPrivate(const QString &host #ifndef QT_NO_NETWORKPROXY , networkProxy(QNetworkProxy::NoProxy) #endif + { } @@ -205,12 +207,19 @@ void QHttpNetworkConnectionPrivate::prepareRequest(HttpMessagePair &messagePair) // add missing fields for the request QByteArray value; // check if Content-Length is provided - QIODevice *data = request.data(); - if (data && request.contentLength() == -1) { - if (!data->isSequential()) - request.setContentLength(data->size()); - else - bufferData(messagePair); // ### or do chunked upload + QNonContiguousByteDevice* uploadByteDevice = request.uploadByteDevice(); + if (uploadByteDevice) { + if (request.contentLength() != -1 && uploadByteDevice->size() != -1) { + // both values known, take the smaller one. + request.setContentLength(qMin(uploadByteDevice->size(), request.contentLength())); + } else if (request.contentLength() == -1 && uploadByteDevice->size() != -1) { + // content length not supplied by user, but the upload device knows it + request.setContentLength(uploadByteDevice->size()); + } else if (request.contentLength() != -1 && uploadByteDevice->size() == -1) { + // everything OK, the user supplied us the contentLength + } else if (request.contentLength() == -1 && uploadByteDevice->size() == -1) { + qFatal("QHttpNetworkConnectionPrivate: Neither content-length nor upload device size were given"); + } } // set the Connection/Proxy-Connection: Keep-Alive headers #ifndef QT_NO_NETWORKPROXY @@ -361,18 +370,12 @@ bool QHttpNetworkConnectionPrivate::sendRequest(QAbstractSocket *socket) false); #endif socket->write(header); - QIODevice *data = channels[i].request.d->data; - QHttpNetworkReply *reply = channels[i].reply; - if (reply && reply->d_func()->requestDataBuffer.size()) - data = &channels[i].reply->d_func()->requestDataBuffer; - if (data && (data->isOpen() || data->open(QIODevice::ReadOnly))) { - if (data->isSequential()) { - channels[i].bytesTotal = -1; - QObject::connect(data, SIGNAL(readyRead()), q, SLOT(_q_dataReadyReadNoBuffer())); - QObject::connect(data, SIGNAL(readChannelFinished()), q, SLOT(_q_dataReadyReadNoBuffer())); - } else { - channels[i].bytesTotal = data->size(); - } + QNonContiguousByteDevice* uploadByteDevice = channels[i].request.uploadByteDevice(); + if (uploadByteDevice) { + // connect the signals so this function gets called again + QObject::connect(uploadByteDevice, SIGNAL(readyRead()), q, SLOT(_q_uploadDataReadyRead())); + + channels[i].bytesTotal = channels[i].request.contentLength(); } else { channels[i].state = WaitingState; break; @@ -380,30 +383,81 @@ bool QHttpNetworkConnectionPrivate::sendRequest(QAbstractSocket *socket) // write the initial chunk together with the headers // fall through } - case WritingState: { // write the data - QIODevice *data = channels[i].request.d->data; - if (channels[i].reply->d_func()->requestDataBuffer.size()) - data = &channels[i].reply->d_func()->requestDataBuffer; - if (!data || channels[i].bytesTotal == channels[i].written) { + case WritingState: + { + // write the data + QNonContiguousByteDevice* uploadByteDevice = channels[i].request.uploadByteDevice(); + if (!uploadByteDevice || channels[i].bytesTotal == channels[i].written) { + if (uploadByteDevice) + emit channels[i].reply->dataSendProgress(channels[i].written, channels[i].bytesTotal); channels[i].state = WaitingState; // now wait for response + sendRequest(socket); break; } - QByteArray chunk; - chunk.resize(ChunkSize); - qint64 readSize = data->read(chunk.data(), ChunkSize); - if (readSize == -1) { - // source has reached EOF - channels[i].state = WaitingState; // now wait for response - } else if (readSize > 0) { - // source gave us something useful - channels[i].written += socket->write(chunk.data(), readSize); - if (channels[i].reply) - emit channels[i].reply->dataSendProgress(channels[i].written, channels[i].bytesTotal); + // only feed the QTcpSocket buffer when there is less than 32 kB in it + const qint64 socketBufferFill = 32*1024; + const qint64 socketWriteMaxSize = 16*1024; + + +#ifndef QT_NO_OPENSSL + QSslSocket *sslSocket = qobject_cast<QSslSocket*>(socket); + while ((sslSocket->encryptedBytesToWrite() + sslSocket->bytesToWrite()) <= socketBufferFill + && channels[i].bytesTotal != channels[i].written) +#else + while (socket->bytesToWrite() <= socketBufferFill + && channels[i].bytesTotal != channels[i].written) +#endif + { + // get pointer to upload data + qint64 currentReadSize; + qint64 desiredReadSize = qMin(socketWriteMaxSize, channels[i].bytesTotal - channels[i].written); + const char *readPointer = uploadByteDevice->readPointer(desiredReadSize, currentReadSize); + + if (currentReadSize == -1) { + // premature eof happened + emitReplyError(socket, channels[i].reply, QNetworkReply::UnknownNetworkError); + return false; + break; + } else if (readPointer == 0 || currentReadSize == 0) { + // nothing to read currently, break the loop + break; + } else { + qint64 currentWriteSize = socket->write(readPointer, currentReadSize); + if (currentWriteSize == -1 || currentWriteSize != currentReadSize) { + // socket broke down + emitReplyError(socket, channels[i].reply, QNetworkReply::UnknownNetworkError); + return false; + } else { + channels[i].written += currentWriteSize; + uploadByteDevice->advanceReadPointer(currentWriteSize); + + emit channels[i].reply->dataSendProgress(channels[i].written, channels[i].bytesTotal); + + if (channels[i].written == channels[i].bytesTotal) { + // make sure this function is called once again + channels[i].state = WaitingState; + sendRequest(socket); + break; + } + } + } } break; } + case WaitingState: + { + QNonContiguousByteDevice* uploadByteDevice = channels[i].request.uploadByteDevice(); + if (uploadByteDevice) { + QObject::disconnect(uploadByteDevice, SIGNAL(readyRead()), q, SLOT(_q_uploadDataReadyRead())); + } + // ensure we try to receive a reply in all cases, even if _q_readyRead_ hat not been called + // this is needed if the sends an reply before we have finished sending the request. In that + // case receiveReply had been called before but ignored the server reply + receiveReply(socket, channels[i].reply); + break; + } case ReadingState: case Wait4AuthState: // ignore _q_bytesWritten in these states @@ -479,6 +533,9 @@ bool QHttpNetworkConnectionPrivate::expand(QAbstractSocket *socket, QHttpNetwork // make sure that the reply is valid if (channels[i].reply != reply) return true; + // emit dataReadProgress signal (signal is currently not connected + // to the rest of QNAM) since readProgress of the + // QNonContiguousByteDevice is used emit reply->dataReadProgress(reply->d_func()->totalProgress, 0); // make sure that the reply is valid if (channels[i].reply != reply) @@ -529,10 +586,20 @@ void QHttpNetworkConnectionPrivate::receiveReply(QAbstractSocket *socket, QHttpN QHttpNetworkReplyPrivate::ReplyState state = reply ? reply->d_func()->state : QHttpNetworkReplyPrivate::AllDoneState; switch (state) { case QHttpNetworkReplyPrivate::NothingDoneState: - case QHttpNetworkReplyPrivate::ReadingStatusState: - bytes += reply->d_func()->readStatus(socket); + case QHttpNetworkReplyPrivate::ReadingStatusState: { + qint64 statusBytes = reply->d_func()->readStatus(socket); + if (statusBytes == -1) { + // error reading the status, close the socket and emit error + socket->close(); + reply->d_func()->errorString = errorDetail(QNetworkReply::ProtocolFailure, socket); + emit reply->finishedWithError(QNetworkReply::ProtocolFailure, reply->d_func()->errorString); + QMetaObject::invokeMethod(q, "_q_startNextRequest", Qt::QueuedConnection); + break; + } + bytes += statusBytes; channels[i].lastStatus = reply->d_func()->statusCode; break; + } case QHttpNetworkReplyPrivate::ReadingHeaderState: bytes += reply->d_func()->readHeader(socket); if (reply->d_func()->state == QHttpNetworkReplyPrivate::ReadingDataState) { @@ -569,6 +636,9 @@ void QHttpNetworkConnectionPrivate::receiveReply(QAbstractSocket *socket, QHttpN // make sure that the reply is valid if (channels[i].reply != reply) return; + // emit dataReadProgress signal (signal is currently not connected + // to the rest of QNAM) since readProgress of the + // QNonContiguousByteDevice is used emit reply->dataReadProgress(reply->d_func()->totalProgress, reply->d_func()->bodyLength); // make sure that the reply is valid if (channels[i].reply != reply) @@ -635,8 +705,25 @@ void QHttpNetworkConnectionPrivate::handleStatus(QAbstractSocket *socket, QHttpN case 407: handleAuthenticateChallenge(socket, reply, (statusCode == 407), resend); if (resend) { + int i = indexOf(socket); + + QNonContiguousByteDevice* uploadByteDevice = channels[i].request.uploadByteDevice(); + if (uploadByteDevice) { + if (uploadByteDevice->reset()) { + channels[i].written = 0; + } else { + emitReplyError(socket, reply, QNetworkReply::ContentReSendError); + break; + } + } + eraseData(reply); - sendRequest(socket); + + // also use async _q_startNextRequest so we dont break with closed + // proxy or server connections.. + channels[i].resendCurrent = true; + QMetaObject::invokeMethod(q, "_q_startNextRequest", Qt::QueuedConnection); + } break; default: @@ -970,6 +1057,7 @@ void QHttpNetworkConnectionPrivate::_q_bytesWritten(qint64 bytes) QAbstractSocket *socket = qobject_cast<QAbstractSocket*>(q->sender()); if (!socket) return; // ### error + // bytes have been written to the socket. write even more of them :) if (isSocketWriting(socket)) sendRequest(socket); // otherwise we do nothing @@ -1128,80 +1216,21 @@ void QHttpNetworkConnectionPrivate::_q_proxyAuthenticationRequired(const QNetwor } #endif -void QHttpNetworkConnectionPrivate::_q_dataReadyReadNoBuffer() +void QHttpNetworkConnectionPrivate::_q_uploadDataReadyRead() { Q_Q(QHttpNetworkConnection); - // data emitted either readyRead() + // upload data emitted readyRead() // find out which channel it is for - QIODevice *sender = qobject_cast<QIODevice *>(q->sender()); + QObject *sender = q->sender(); - // won't match anything if the qobject_cast above failed for (int i = 0; i < channelCount; ++i) { - if (sender == channels[i].request.data()) { + if (sender == channels[i].request.uploadByteDevice()) { sendRequest(channels[i].socket); break; } } } -void QHttpNetworkConnectionPrivate::_q_dataReadyReadBuffer() -{ - Q_Q(QHttpNetworkConnection); - QIODevice *sender = qobject_cast<QIODevice *>(q->sender()); - HttpMessagePair *thePair = 0; - for (int i = 0; !thePair && i < lowPriorityQueue.size(); ++i) - if (lowPriorityQueue.at(i).first.data() == sender) - thePair = &lowPriorityQueue[i]; - - for (int i = 0; !thePair && i < highPriorityQueue.size(); ++i) - if (highPriorityQueue.at(i).first.data() == sender) - thePair = &highPriorityQueue[i]; - - if (thePair) { - bufferData(*thePair); - - // are we finished buffering? - if (!thePair->second->d_func()->requestIsBuffering) - _q_startNextRequest(); - } -} - -void QHttpNetworkConnectionPrivate::bufferData(HttpMessagePair &messagePair) -{ - Q_Q(QHttpNetworkConnection); - QHttpNetworkRequest &request = messagePair.first; - QHttpNetworkReply *reply = messagePair.second; - Q_ASSERT(request.data()); - if (!reply->d_func()->requestIsBuffering) { // first time - QObject::connect(request.data(), SIGNAL(readyRead()), q, SLOT(_q_dataReadyReadBuffer())); - QObject::connect(request.data(), SIGNAL(readChannelFinished()), q, SLOT(_q_dataReadyReadBuffer())); - reply->d_func()->requestIsBuffering = true; - reply->d_func()->requestDataBuffer.open(QIODevice::ReadWrite); - } - - // always try to read at least one byte - // ### FIXME! use a QRingBuffer - qint64 bytesToRead = qMax<qint64>(1, request.data()->bytesAvailable()); - QByteArray newData; - newData.resize(bytesToRead); - qint64 bytesActuallyRead = request.data()->read(newData.data(), bytesToRead); - - if (bytesActuallyRead > 0) { - // we read something - newData.chop(bytesToRead - bytesActuallyRead); - reply->d_func()->requestDataBuffer.write(newData); - } else if (bytesActuallyRead == -1) { // last time - QObject::disconnect(request.data(), SIGNAL(readyRead()), q, SLOT(_q_dataReadyReadBuffer())); - QObject::disconnect(request.data(), SIGNAL(readChannelFinished()), q, SLOT(_q_dataReadyReadBuffer())); - - request.setContentLength(reply->d_func()->requestDataBuffer.size()); - reply->d_func()->requestDataBuffer.seek(0); - reply->d_func()->requestIsBuffering = false; - } -} - -// QHttpNetworkConnection - QHttpNetworkConnection::QHttpNetworkConnection(const QString &hostName, quint16 port, bool encrypt, QObject *parent) : QObject(*(new QHttpNetworkConnectionPrivate(hostName, port, encrypt)), parent) { diff --git a/src/network/access/qhttpnetworkconnection_p.h b/src/network/access/qhttpnetworkconnection_p.h index 09bd459..9b127dd 100644 --- a/src/network/access/qhttpnetworkconnection_p.h +++ b/src/network/access/qhttpnetworkconnection_p.h @@ -146,8 +146,7 @@ private: #ifndef QT_NO_NETWORKPROXY Q_PRIVATE_SLOT(d_func(), void _q_proxyAuthenticationRequired(const QNetworkProxy&, QAuthenticator*)) #endif - Q_PRIVATE_SLOT(d_func(), void _q_dataReadyReadBuffer()) - Q_PRIVATE_SLOT(d_func(), void _q_dataReadyReadNoBuffer()) + Q_PRIVATE_SLOT(d_func(), void _q_uploadDataReadyRead()) #ifndef QT_NO_OPENSSL Q_PRIVATE_SLOT(d_func(), void _q_encrypted()) @@ -209,8 +208,8 @@ public: #ifndef QT_NO_NETWORKPROXY void _q_proxyAuthenticationRequired(const QNetworkProxy &proxy, QAuthenticator *auth); // from transparent proxy #endif - void _q_dataReadyReadNoBuffer(); - void _q_dataReadyReadBuffer(); + + void _q_uploadDataReadyRead(); void createAuthorization(QAbstractSocket *socket, QHttpNetworkRequest &request); bool ensureConnection(QAbstractSocket *socket); @@ -219,7 +218,6 @@ public: #ifndef QT_NO_COMPRESS bool expand(QAbstractSocket *socket, QHttpNetworkReply *reply, bool dataComplete); #endif - void bufferData(HttpMessagePair &request); void removeReply(QHttpNetworkReply *reply); QString hostName; diff --git a/src/network/access/qhttpnetworkreply.cpp b/src/network/access/qhttpnetworkreply.cpp index fe3f6af..310994c 100644 --- a/src/network/access/qhttpnetworkreply.cpp +++ b/src/network/access/qhttpnetworkreply.cpp @@ -409,39 +409,62 @@ qint64 QHttpNetworkReplyPrivate::readStatus(QAbstractSocket *socket) if (fragment.endsWith('\r')) { fragment.truncate(fragment.length()-1); } - parseStatus(fragment); + bool ok = parseStatus(fragment); state = ReadingHeaderState; fragment.clear(); // next fragment + + if (!ok) + return -1; break; } else { c = 0; bytes += socket->read(&c, 1); fragment.append(c); } + + // is this a valid reply? + if (fragment.length() >= 5 && !fragment.startsWith("HTTP/")) + return -1; + } + return bytes; } -void QHttpNetworkReplyPrivate::parseStatus(const QByteArray &status) +bool QHttpNetworkReplyPrivate::parseStatus(const QByteArray &status) { - const QByteArrayMatcher sp(" "); - int i = sp.indexIn(status); - const QByteArray version = status.mid(0, i); - int j = sp.indexIn(status, i + 1); + // from RFC 2616: + // Status-Line = HTTP-Version SP Status-Code SP Reason-Phrase CRLF + // HTTP-Version = "HTTP" "/" 1*DIGIT "." 1*DIGIT + // that makes: 'HTTP/n.n xxx Message' + // byte count: 0123456789012 + + static const int minLength = 11; + static const int dotPos = 6; + static const int spacePos = 8; + static const char httpMagic[] = "HTTP/"; + + if (status.length() < minLength + || !status.startsWith(httpMagic) + || status.at(dotPos) != '.' + || status.at(spacePos) != ' ') { + // I don't know how to parse this status line + return false; + } + + // optimize for the valid case: defer checking until the end + majorVersion = status.at(dotPos - 1) - '0'; + minorVersion = status.at(dotPos + 1) - '0'; + + int i = spacePos; + int j = status.indexOf(' ', i + 1); // j == -1 || at(j) == ' ' so j+1 == 0 && j+1 <= length() const QByteArray code = status.mid(i + 1, j - i - 1); - const QByteArray reason = status.mid(j + 1, status.count() - j); - const QByteArrayMatcher slash("/"); - int k = slash.indexIn(version); - const QByteArrayMatcher dot("."); - int l = dot.indexIn(version, k); - const QByteArray major = version.mid(k + 1, l - k - 1); - const QByteArray minor = version.mid(l + 1, version.count() - l); + bool ok; + statusCode = code.toInt(&ok); + reasonPhrase = QString::fromLatin1(status.constData() + j + 1); - majorVersion = QString::fromAscii(major.constData()).toInt(); - minorVersion = QString::fromAscii(minor.constData()).toInt(); - statusCode = QString::fromAscii(code.constData()).toInt(); - reasonPhrase = QString::fromAscii(reason.constData()); + return ok && uint(majorVersion) <= 9 && uint(minorVersion) <= 9; } qint64 QHttpNetworkReplyPrivate::readHeader(QAbstractSocket *socket) @@ -521,13 +544,13 @@ qint64 QHttpNetworkReplyPrivate::readBody(QAbstractSocket *socket, QIODevice *ou { qint64 bytes = 0; if (isChunked()) { - bytes += transferChunked(socket, out); // chunked transfer encoding (rfc 2616, sec 3.6) + bytes += readReplyBodyChunked(socket, out); // chunked transfer encoding (rfc 2616, sec 3.6) } else if (bodyLength > 0) { // we have a Content-Length - bytes += transferRaw(socket, out, bodyLength - contentRead); + bytes += readReplyBodyRaw(socket, out, bodyLength - contentRead); if (contentRead + bytes == bodyLength) state = AllDoneState; } else { - bytes += transferRaw(socket, out, socket->bytesAvailable()); + bytes += readReplyBodyRaw(socket, out, socket->bytesAvailable()); } if (state == AllDoneState) socket->readAll(); // Read the rest to clean (CRLF) @@ -535,7 +558,7 @@ qint64 QHttpNetworkReplyPrivate::readBody(QAbstractSocket *socket, QIODevice *ou return bytes; } -qint64 QHttpNetworkReplyPrivate::transferRaw(QIODevice *in, QIODevice *out, qint64 size) +qint64 QHttpNetworkReplyPrivate::readReplyBodyRaw(QIODevice *in, QIODevice *out, qint64 size) { qint64 bytes = 0; Q_ASSERT(in); @@ -561,7 +584,7 @@ qint64 QHttpNetworkReplyPrivate::transferRaw(QIODevice *in, QIODevice *out, qint } -qint64 QHttpNetworkReplyPrivate::transferChunked(QIODevice *in, QIODevice *out) +qint64 QHttpNetworkReplyPrivate::readReplyBodyChunked(QIODevice *in, QIODevice *out) { qint64 bytes = 0; while (in->bytesAvailable()) { // while we can read from input @@ -660,4 +683,4 @@ void QHttpNetworkReply::ignoreSslErrors() QT_END_NAMESPACE -#endif // QT_NO_HTTP
\ No newline at end of file +#endif // QT_NO_HTTP diff --git a/src/network/access/qhttpnetworkreply_p.h b/src/network/access/qhttpnetworkreply_p.h index c17c65c..cc5cce8 100644 --- a/src/network/access/qhttpnetworkreply_p.h +++ b/src/network/access/qhttpnetworkreply_p.h @@ -139,7 +139,7 @@ Q_SIGNALS: void finishedWithError(QNetworkReply::NetworkError errorCode, const QString &detail = QString()); void headerChanged(); void dataReadProgress(int done, int total); - void dataSendProgress(int done, int total); + void dataSendProgress(qint64 done, qint64 total); private: Q_DECLARE_PRIVATE(QHttpNetworkReply) @@ -154,7 +154,7 @@ public: QHttpNetworkReplyPrivate(const QUrl &newUrl = QUrl()); ~QHttpNetworkReplyPrivate(); qint64 readStatus(QAbstractSocket *socket); - void parseStatus(const QByteArray &status); + bool parseStatus(const QByteArray &status); qint64 readHeader(QAbstractSocket *socket); void parseHeader(const QByteArray &header); qint64 readBody(QAbstractSocket *socket, QIODevice *out); @@ -162,8 +162,8 @@ public: QAuthenticatorPrivate::Method authenticationMethod(bool isProxy) const; void clear(); - qint64 transferRaw(QIODevice *in, QIODevice *out, qint64 size); - qint64 transferChunked(QIODevice *in, QIODevice *out); + qint64 readReplyBodyRaw(QIODevice *in, QIODevice *out, qint64 size); + qint64 readReplyBodyChunked(QIODevice *in, QIODevice *out); qint64 getChunkSize(QIODevice *in, qint64 *chunkSize); qint64 bytesAvailable() const; @@ -206,7 +206,6 @@ public: QByteArray responseData; // uncompressed body QByteArray compressedData; // compressed body (temporary) - QBuffer requestDataBuffer; bool requestIsBuffering; bool requestIsPrepared; }; diff --git a/src/network/access/qhttpnetworkrequest.cpp b/src/network/access/qhttpnetworkrequest.cpp index 420cb69..7df68fc 100644 --- a/src/network/access/qhttpnetworkrequest.cpp +++ b/src/network/access/qhttpnetworkrequest.cpp @@ -40,12 +40,13 @@ ****************************************************************************/ #include "qhttpnetworkrequest_p.h" +#include "private/qnoncontiguousbytedevice_p.h" QT_BEGIN_NAMESPACE QHttpNetworkRequestPrivate::QHttpNetworkRequestPrivate(QHttpNetworkRequest::Operation op, QHttpNetworkRequest::Priority pri, const QUrl &newUrl) - : QHttpNetworkHeaderPrivate(newUrl), operation(op), priority(pri), data(0), + : QHttpNetworkHeaderPrivate(newUrl), operation(op), priority(pri), uploadByteDevice(0), autoDecompress(false) { } @@ -55,7 +56,7 @@ QHttpNetworkRequestPrivate::QHttpNetworkRequestPrivate(const QHttpNetworkRequest { operation = other.operation; priority = other.priority; - data = other.data; + uploadByteDevice = other.uploadByteDevice; autoDecompress = other.autoDecompress; } @@ -67,7 +68,7 @@ bool QHttpNetworkRequestPrivate::operator==(const QHttpNetworkRequestPrivate &ot { return QHttpNetworkHeaderPrivate::operator==(other) && (operation == other.operation) - && (data == other.data); + && (uploadByteDevice == other.uploadByteDevice); } QByteArray QHttpNetworkRequestPrivate::methodName() const @@ -109,7 +110,7 @@ QByteArray QHttpNetworkRequestPrivate::uri(bool throughProxy) const QUrl::FormattingOptions format(QUrl::RemoveFragment); // for POST, query data is send as content - if (operation == QHttpNetworkRequest::Post && !data) + if (operation == QHttpNetworkRequest::Post && !uploadByteDevice) format |= QUrl::RemoveQuery; // for requests through proxy, the Request-URI contains full url if (throughProxy) @@ -140,7 +141,7 @@ QByteArray QHttpNetworkRequestPrivate::header(const QHttpNetworkRequest &request // add content type, if not set in the request if (request.headerField("content-type").isEmpty()) ba += "Content-Type: application/x-www-form-urlencoded\r\n"; - if (!request.d->data && request.d->url.hasQuery()) { + if (!request.d->uploadByteDevice && request.d->url.hasQuery()) { QByteArray query = request.d->url.encodedQuery(); ba += "Content-Length: "+ QByteArray::number(query.size()) + "\r\n"; ba += "\r\n"; @@ -236,14 +237,14 @@ void QHttpNetworkRequest::setPriority(Priority priority) d->priority = priority; } -QIODevice *QHttpNetworkRequest::data() const +void QHttpNetworkRequest::setUploadByteDevice(QNonContiguousByteDevice *bd) { - return d->data; + d->uploadByteDevice = bd; } -void QHttpNetworkRequest::setData(QIODevice *data) +QNonContiguousByteDevice* QHttpNetworkRequest::uploadByteDevice() const { - d->data = data; + return d->uploadByteDevice; } int QHttpNetworkRequest::majorVersion() const diff --git a/src/network/access/qhttpnetworkrequest_p.h b/src/network/access/qhttpnetworkrequest_p.h index d18e116..ed4325a 100644 --- a/src/network/access/qhttpnetworkrequest_p.h +++ b/src/network/access/qhttpnetworkrequest_p.h @@ -58,6 +58,8 @@ QT_BEGIN_NAMESPACE +class QNonContiguousByteDevice; + class QHttpNetworkRequestPrivate; class Q_AUTOTEST_EXPORT QHttpNetworkRequest: public QHttpNetworkHeader { @@ -104,8 +106,8 @@ public: Priority priority() const; void setPriority(Priority priority); - QIODevice *data() const; - void setData(QIODevice *data); + void setUploadByteDevice(QNonContiguousByteDevice *bd); + QNonContiguousByteDevice* uploadByteDevice() const; private: QSharedDataPointer<QHttpNetworkRequestPrivate> d; @@ -113,7 +115,6 @@ private: friend class QHttpNetworkConnectionPrivate; }; - class QHttpNetworkRequestPrivate : public QHttpNetworkHeaderPrivate { public: @@ -129,7 +130,7 @@ public: QHttpNetworkRequest::Operation operation; QHttpNetworkRequest::Priority priority; - mutable QIODevice *data; + mutable QNonContiguousByteDevice* uploadByteDevice; bool autoDecompress; }; diff --git a/src/network/access/qnetworkaccessbackend.cpp b/src/network/access/qnetworkaccessbackend.cpp index df468b8..b9d1b85 100644 --- a/src/network/access/qnetworkaccessbackend.cpp +++ b/src/network/access/qnetworkaccessbackend.cpp @@ -50,6 +50,8 @@ #include "qnetworkaccesscachebackend_p.h" #include "qabstractnetworkcache.h" +#include "private/qnoncontiguousbytedevice_p.h" + QT_BEGIN_NAMESPACE static bool factoryDataShutdown = false; @@ -109,17 +111,43 @@ QNetworkAccessBackend *QNetworkAccessManagerPrivate::findBackend(QNetworkAccessM return 0; } -QNetworkAccessBackend::QNetworkAccessBackend() + +QNonContiguousByteDevice* QNetworkAccessBackend::createUploadByteDevice() { + QNonContiguousByteDevice* device = 0; + + if (reply->outgoingDataBuffer) + device = QNonContiguousByteDeviceFactory::create(reply->outgoingDataBuffer); + else + device = QNonContiguousByteDeviceFactory::create(reply->outgoingData); + + bool bufferDisallowed = + reply->request.attribute(QNetworkRequest::DoNotBufferUploadDataAttribute, + QVariant(false)) == QVariant(true); + if (bufferDisallowed) + device->disableReset(); + + // make sure we delete this later + device->setParent(this); + + connect(device, SIGNAL(readProgress(qint64,qint64)), this, SLOT(emitReplyUploadProgress(qint64,qint64))); + + return device; } -QNetworkAccessBackend::~QNetworkAccessBackend() +// need to have this function since the reply is a private member variable +// and the special backends need to access this. +void QNetworkAccessBackend::emitReplyUploadProgress(qint64 bytesSent, qint64 bytesTotal) { + reply->emitUploadProgress(bytesSent, bytesTotal); } -void QNetworkAccessBackend::upstreamReadyRead() +QNetworkAccessBackend::QNetworkAccessBackend() +{ +} + +QNetworkAccessBackend::~QNetworkAccessBackend() { - // do nothing } void QNetworkAccessBackend::downstreamReadyWrite() @@ -184,23 +212,6 @@ bool QNetworkAccessBackend::isCachingEnabled() const return reply->isCachingEnabled(); } -qint64 QNetworkAccessBackend::upstreamBytesAvailable() const -{ - return reply->writeBuffer.size(); -} - -void QNetworkAccessBackend::upstreamBytesConsumed(qint64 count) -{ - // remove count bytes from the write buffer - reply->consume(count); -} - -QByteArray QNetworkAccessBackend::readUpstream() -{ - // ### this is expensive. Consider making QRingBuffer::peekAll keep the buffer it returns - return reply->writeBuffer.peek(upstreamBytesAvailable()); -} - qint64 QNetworkAccessBackend::nextDownstreamBlockSize() const { return reply->nextDownstreamBlockSize(); @@ -213,12 +224,12 @@ qint64 QNetworkAccessBackend::downstreamBytesToConsume() const void QNetworkAccessBackend::writeDownstreamData(const QByteArray &data) { - reply->feed(data); + reply->appendDownstreamData(data); } void QNetworkAccessBackend::writeDownstreamData(QIODevice *data) { - reply->feed(data); + reply->appendDownstreamData(data); } QVariant QNetworkAccessBackend::header(QNetworkRequest::KnownHeaders header) const diff --git a/src/network/access/qnetworkaccessbackend_p.h b/src/network/access/qnetworkaccessbackend_p.h index 9012396..6035f3a 100644 --- a/src/network/access/qnetworkaccessbackend_p.h +++ b/src/network/access/qnetworkaccessbackend_p.h @@ -70,6 +70,8 @@ class QNetworkAccessManagerPrivate; class QNetworkReplyImplPrivate; class QAbstractNetworkCache; class QNetworkCacheMetaData; +class QNetworkAccessBackendUploadIODevice; +class QNonContiguousByteDevice; // Should support direct file upload from disk or download to disk. // @@ -86,14 +88,13 @@ public: // have different names. The Connection has two streams: // // - Upstream: - // Upstream is data that is being written into this connection, - // from the user. Upstream operates in a "pull" mechanism: the - // connection will be notified that there is more data available - // by a call to "upstreamReadyRead". The number of bytes - // available is given by upstreamBytesAvailable(). A call to - // readUpstream() always yields the entire upstream buffer. When - // the connection has processed a certain amount of bytes from - // that buffer, it should call upstreamBytesConsumed(). + // The upstream uses a QNonContiguousByteDevice provided + // by the backend. This device emits the usual readyRead() + // signal when the backend has data available for the connection + // to write. The different backends can listen on this signal + // and then pull upload data from the QNonContiguousByteDevice and + // deal with it. + // // // - Downstream: // Downstream is the data that is being read from this @@ -111,12 +112,9 @@ public: virtual void open() = 0; virtual void closeDownstreamChannel() = 0; - virtual void closeUpstreamChannel() = 0; virtual bool waitForDownstreamReadyRead(int msecs) = 0; - virtual bool waitForUpstreamBytesWritten(int msecs) = 0; // slot-like: - virtual void upstreamReadyRead(); virtual void downstreamReadyWrite(); virtual void copyFinished(QIODevice *); virtual void ignoreSslErrors(); @@ -155,18 +153,24 @@ public: QVariant attribute(QNetworkRequest::Attribute code) const; void setAttribute(QNetworkRequest::Attribute code, const QVariant &value); + // return true if the QNonContiguousByteDevice of the upload + // data needs to support reset(). Currently needed for HTTP. + // This will possibly enable buffering of the upload data. + virtual bool needsResetableUploadData() {return false;}; + protected: - // these functions control the upstream mechanism - // that is, data coming into the backend and out via the connection - qint64 upstreamBytesAvailable() const; - void upstreamBytesConsumed(qint64 count); - QByteArray readUpstream(); + // Create the device used for reading the upload data + QNonContiguousByteDevice* createUploadByteDevice(); + // these functions control the downstream mechanism // that is, data that has come via the connection and is going out the backend qint64 nextDownstreamBlockSize() const; qint64 downstreamBytesToConsume() const; void writeDownstreamData(const QByteArray &data); + +public slots: + // for task 251801, needs to be a slot to be called asynchronously void writeDownstreamData(QIODevice *data); protected slots: @@ -179,10 +183,12 @@ protected slots: void metaDataChanged(); void redirectionRequested(const QUrl &destination); void sslErrors(const QList<QSslError> &errors); + void emitReplyUploadProgress(qint64 bytesSent, qint64 bytesTotal); private: friend class QNetworkAccessManager; friend class QNetworkAccessManagerPrivate; + friend class QNetworkAccessBackendUploadIODevice; QNetworkAccessManagerPrivate *manager; QNetworkReplyImplPrivate *reply; }; diff --git a/src/network/access/qnetworkaccessdebugpipebackend.cpp b/src/network/access/qnetworkaccessdebugpipebackend.cpp index 2e5f1b1..d4bda9a 100644 --- a/src/network/access/qnetworkaccessdebugpipebackend.cpp +++ b/src/network/access/qnetworkaccessdebugpipebackend.cpp @@ -41,6 +41,8 @@ #include "qnetworkaccessdebugpipebackend_p.h" #include "QtCore/qdatastream.h" +#include <QCoreApplication> +#include "private/qnoncontiguousbytedevice_p.h" QT_BEGIN_NAMESPACE @@ -51,12 +53,6 @@ enum { WriteBufferSize = ReadBufferSize }; -struct QNetworkAccessDebugPipeBackend::DataPacket -{ - QList<QPair<QByteArray, QByteArray> > headers; - QByteArray data; -}; - QNetworkAccessBackend * QNetworkAccessDebugPipeBackendFactory::create(QNetworkAccessManager::Operation op, const QNetworkRequest &request) const @@ -79,12 +75,14 @@ QNetworkAccessDebugPipeBackendFactory::create(QNetworkAccessManager::Operation o } QNetworkAccessDebugPipeBackend::QNetworkAccessDebugPipeBackend() - : incomingPacketSize(0), bareProtocol(false) + : bareProtocol(false), hasUploadFinished(false), hasDownloadFinished(false), + hasEverythingFinished(false), bytesDownloaded(0), bytesUploaded(0) { } QNetworkAccessDebugPipeBackend::~QNetworkAccessDebugPipeBackend() { + // this is signals disconnect, not network! socket.disconnect(this); // we're not interested in the signals at this point } @@ -92,160 +90,150 @@ void QNetworkAccessDebugPipeBackend::open() { socket.connectToHost(url().host(), url().port(12345)); socket.setReadBufferSize(ReadBufferSize); + + // socket ready read -> we can push from socket to downstream connect(&socket, SIGNAL(readyRead()), SLOT(socketReadyRead())); - connect(&socket, SIGNAL(bytesWritten(qint64)), SLOT(socketBytesWritten(qint64))); connect(&socket, SIGNAL(error(QAbstractSocket::SocketError)), SLOT(socketError())); connect(&socket, SIGNAL(disconnected()), SLOT(socketDisconnected())); + connect(&socket, SIGNAL(connected()), SLOT(socketConnected())); + // socket bytes written -> we can push more from upstream to socket + connect(&socket, SIGNAL(bytesWritten(qint64)), SLOT(socketBytesWritten(qint64))); bareProtocol = url().queryItemValue(QLatin1String("bare")) == QLatin1String("1"); - if (!bareProtocol) { - // "Handshake": - // send outgoing metadata and the URL being requested - DataPacket packet; - //packet.metaData = request().metaData(); - packet.data = url().toEncoded(); - send(packet); + if (operation() == QNetworkAccessManager::PutOperation) { + uploadByteDevice = createUploadByteDevice(); + QObject::connect(uploadByteDevice, SIGNAL(readyRead()), this, SLOT(uploadReadyReadSlot())); + QMetaObject::invokeMethod(this, "uploadReadyReadSlot", Qt::QueuedConnection); } } -void QNetworkAccessDebugPipeBackend::closeDownstreamChannel() +void QNetworkAccessDebugPipeBackend::socketReadyRead() { - if (operation() == QNetworkAccessManager::GetOperation) - socket.disconnectFromHost(); + pushFromSocketToDownstream(); } -void QNetworkAccessDebugPipeBackend::closeUpstreamChannel() +void QNetworkAccessDebugPipeBackend::downstreamReadyWrite() { - if (operation() == QNetworkAccessManager::PutOperation) - socket.disconnectFromHost(); - else if (operation() == QNetworkAccessManager::PostOperation) { - send(DataPacket()); - } + pushFromSocketToDownstream(); } -bool QNetworkAccessDebugPipeBackend::waitForDownstreamReadyRead(int ms) +void QNetworkAccessDebugPipeBackend::socketBytesWritten(qint64) { - readyReadEmitted = false; - if (socket.bytesAvailable()) { - socketReadyRead(); - if (readyReadEmitted) - return true; - } - socket.waitForReadyRead(ms); - return readyReadEmitted; + pushFromUpstreamToSocket(); } -bool QNetworkAccessDebugPipeBackend::waitForUpstreamBytesWritten(int ms) +void QNetworkAccessDebugPipeBackend::uploadReadyReadSlot() { - bytesWrittenEmitted = false; - upstreamReadyRead(); - if (bytesWrittenEmitted) - return true; - - socket.waitForBytesWritten(ms); - return bytesWrittenEmitted; + pushFromUpstreamToSocket(); } -void QNetworkAccessDebugPipeBackend::upstreamReadyRead() +void QNetworkAccessDebugPipeBackend::pushFromSocketToDownstream() { - int maxWrite = WriteBufferSize - socket.bytesToWrite(); - if (maxWrite <= 0) - return; // can't write yet, wait for the socket to write - - if (bareProtocol) { - QByteArray data = readUpstream(); - if (data.isEmpty()) - return; + QByteArray buffer; - socket.write(data); - upstreamBytesConsumed(data.size()); - bytesWrittenEmitted = true; + if (socket.state() == QAbstractSocket::ConnectingState) { return; } - DataPacket packet; - packet.data = readUpstream(); - if (packet.data.isEmpty()) - return; // we'll be called again when there's data - if (packet.data.size() > maxWrite) - packet.data.truncate(maxWrite); - - if (!send(packet)) { - QString msg = QObject::tr("Write error writing to %1: %2") - .arg(url().toString(), socket.errorString()); - error(QNetworkReply::ProtocolFailure, msg); + forever { + if (hasDownloadFinished) + return; - finished(); - return; + buffer.resize(ReadBufferSize); + qint64 haveRead = socket.read(buffer.data(), ReadBufferSize); + + if (haveRead == -1) { + hasDownloadFinished = true; + // this ensures a good last downloadProgress is emitted + setHeader(QNetworkRequest::ContentLengthHeader, QVariant()); + possiblyFinish(); + break; + } else if (haveRead == 0) { + break; + } else { + // have read something + buffer.resize(haveRead); + bytesDownloaded += haveRead; + writeDownstreamData(buffer); + } } - upstreamBytesConsumed(packet.data.size()); - bytesWrittenEmitted = true; } -void QNetworkAccessDebugPipeBackend::downstreamReadyWrite() +void QNetworkAccessDebugPipeBackend::pushFromUpstreamToSocket() { - socketReadyRead(); -} + // FIXME + if (operation() == QNetworkAccessManager::PutOperation) { + if (hasUploadFinished) + return; -void QNetworkAccessDebugPipeBackend::socketReadyRead() -{ - if (bareProtocol) { - qint64 bytesToRead = socket.bytesAvailable(); - if (bytesToRead) { - QByteArray buffer; - buffer.resize(bytesToRead); - qint64 bytesRead = socket.read(buffer.data(), bytesToRead); - if (bytesRead < bytesToRead) - buffer.truncate(bytesRead); - writeDownstreamData(buffer); - readyReadEmitted = true; + forever { + if (socket.bytesToWrite() >= WriteBufferSize) + return; + + qint64 haveRead; + const char *readPointer = uploadByteDevice->readPointer(WriteBufferSize, haveRead); + if (haveRead == -1) { + // EOF + hasUploadFinished = true; + emitReplyUploadProgress(bytesUploaded, bytesUploaded); + possiblyFinish(); + break; + } else if (haveRead == 0 || readPointer == 0) { + // nothing to read right now, we will be called again later + break; + } else { + qint64 haveWritten; + haveWritten = socket.write(readPointer, haveRead); + + if (haveWritten < 0) { + // write error! + QString msg = QCoreApplication::translate("QNetworkAccessDebugPipeBackend", "Write error writing to %1: %2") + .arg(url().toString(), socket.errorString()); + error(QNetworkReply::ProtocolFailure, msg); + finished(); + return; + } else { + uploadByteDevice->advanceReadPointer(haveWritten); + bytesUploaded += haveWritten; + emitReplyUploadProgress(bytesUploaded, -1); + } + + //QCoreApplication::processEvents(); + + } } - return; } +} - while (canReceive() && - (socket.state() == QAbstractSocket::UnconnectedState || nextDownstreamBlockSize())) { - DataPacket packet; - if (receive(packet)) { - if (!packet.headers.isEmpty()) { - QList<QPair<QByteArray, QByteArray> >::ConstIterator - it = packet.headers.constBegin(), - end = packet.headers.constEnd(); - for ( ; it != end; ++it) - setRawHeader(it->first, it->second); - metaDataChanged(); - } +void QNetworkAccessDebugPipeBackend::possiblyFinish() +{ + if (hasEverythingFinished) + return; + hasEverythingFinished = true; - if (!packet.data.isEmpty()) { - writeDownstreamData(packet.data); - readyReadEmitted = true; - } + if ((operation() == QNetworkAccessManager::GetOperation) && hasDownloadFinished) { + socket.close(); + finished(); + } else if ((operation() == QNetworkAccessManager::PutOperation) && hasUploadFinished) { + socket.close(); + finished(); + } - if (packet.headers.isEmpty() && packet.data.isEmpty()) { - // it's an eof - socket.close(); - readyReadEmitted = true; - } - } else { - // got an error - QString msg = QObject::tr("Read error reading from %1: %2") - .arg(url().toString(), socket.errorString()); - error(QNetworkReply::ProtocolFailure, msg); - finished(); - return; - } - } } -void QNetworkAccessDebugPipeBackend::socketBytesWritten(qint64) +void QNetworkAccessDebugPipeBackend::closeDownstreamChannel() { - upstreamReadyRead(); + qWarning() << "QNetworkAccessDebugPipeBackend::closeDownstreamChannel()" << operation(); + //if (operation() == QNetworkAccessManager::GetOperation) + // socket.disconnectFromHost(); } + void QNetworkAccessDebugPipeBackend::socketError() { + qWarning() << "QNetworkAccessDebugPipeBackend::socketError()" << socket.error(); QNetworkReply::NetworkError code; switch (socket.error()) { case QAbstractSocket::RemoteHostClosedError: @@ -269,76 +257,27 @@ void QNetworkAccessDebugPipeBackend::socketError() void QNetworkAccessDebugPipeBackend::socketDisconnected() { - socketReadyRead(); - if (incomingPacketSize == 0 && socket.bytesToWrite() == 0) { + pushFromSocketToDownstream(); + + if (socket.bytesToWrite() == 0) { // normal close - finished(); } else { // abnormal close QString msg = QObject::tr("Remote host closed the connection prematurely on %1") .arg(url().toString()); error(QNetworkReply::RemoteHostClosedError, msg); - finished(); } } -bool QNetworkAccessDebugPipeBackend::send(const DataPacket &packet) -{ - QByteArray ba; - { - QDataStream stream(&ba, QIODevice::WriteOnly); - stream.setVersion(QDataStream::Qt_4_4); - - stream << packet.headers << packet.data; - } - - qint32 outgoingPacketSize = ba.size(); - qint64 written = socket.write((const char*)&outgoingPacketSize, sizeof outgoingPacketSize); - written += socket.write(ba); - return quint64(written) == (outgoingPacketSize + sizeof outgoingPacketSize); -} - -bool QNetworkAccessDebugPipeBackend::receive(DataPacket &packet) +void QNetworkAccessDebugPipeBackend::socketConnected() { - if (!canReceive()) - return false; - - // canReceive() does the setting up for us - Q_ASSERT(socket.bytesAvailable() >= incomingPacketSize); - QByteArray incomingPacket = socket.read(incomingPacketSize); - QDataStream stream(&incomingPacket, QIODevice::ReadOnly); - stream.setVersion(QDataStream::Qt_4_4); - stream >> packet.headers >> packet.data; - - // reset for next packet: - incomingPacketSize = 0; - socket.setReadBufferSize(ReadBufferSize); - return true; } -bool QNetworkAccessDebugPipeBackend::canReceive() +bool QNetworkAccessDebugPipeBackend::waitForDownstreamReadyRead(int ms) { - if (incomingPacketSize == 0) { - // read the packet size - if (quint64(socket.bytesAvailable()) >= sizeof incomingPacketSize) - socket.read((char*)&incomingPacketSize, sizeof incomingPacketSize); - else - return false; - } - - if (incomingPacketSize == 0) { - QString msg = QObject::tr("Protocol error: packet of size 0 received"); - error(QNetworkReply::ProtocolFailure, msg); - finished(); - - socket.blockSignals(true); - socket.abort(); - socket.blockSignals(false); - return false; - } - - return socket.bytesAvailable() >= incomingPacketSize; + qCritical("QNetworkAccess: Debug pipe backend does not support waitForReadyRead()"); + return false; } #endif diff --git a/src/network/access/qnetworkaccessdebugpipebackend_p.h b/src/network/access/qnetworkaccessdebugpipebackend_p.h index 73a35cf..a13edc4 100644 --- a/src/network/access/qnetworkaccessdebugpipebackend_p.h +++ b/src/network/access/qnetworkaccessdebugpipebackend_p.h @@ -66,35 +66,38 @@ class QNetworkAccessDebugPipeBackend: public QNetworkAccessBackend { Q_OBJECT public: - struct DataPacket; QNetworkAccessDebugPipeBackend(); virtual ~QNetworkAccessDebugPipeBackend(); virtual void open(); virtual void closeDownstreamChannel(); - virtual void closeUpstreamChannel(); virtual bool waitForDownstreamReadyRead(int msecs); - virtual bool waitForUpstreamBytesWritten(int msecs); - virtual void upstreamReadyRead(); virtual void downstreamReadyWrite(); +protected: + void pushFromSocketToDownstream(); + void pushFromUpstreamToSocket(); + void possiblyFinish(); + QNonContiguousByteDevice *uploadByteDevice; + private slots: + void uploadReadyReadSlot(); void socketReadyRead(); void socketBytesWritten(qint64 bytes); void socketError(); void socketDisconnected(); + void socketConnected(); private: QTcpSocket socket; - qint32 incomingPacketSize; - bool readyReadEmitted; - bool bytesWrittenEmitted; bool bareProtocol; + bool hasUploadFinished; + bool hasDownloadFinished; + bool hasEverythingFinished; - bool send(const DataPacket &packet); - bool canReceive(); - bool receive(DataPacket &packet); + qint64 bytesDownloaded; + qint64 bytesUploaded; }; class QNetworkAccessDebugPipeBackendFactory: public QNetworkAccessBackendFactory diff --git a/src/network/access/qnetworkaccessfilebackend.cpp b/src/network/access/qnetworkaccessfilebackend.cpp index 8a5a665..6374fde 100644 --- a/src/network/access/qnetworkaccessfilebackend.cpp +++ b/src/network/access/qnetworkaccessfilebackend.cpp @@ -43,6 +43,7 @@ #include "qfileinfo.h" #include "qurlinfo.h" #include "qdir.h" +#include "private/qnoncontiguousbytedevice_p.h" #include <QtCore/QCoreApplication> @@ -77,7 +78,7 @@ QNetworkAccessFileBackendFactory::create(QNetworkAccessManager::Operation op, } QNetworkAccessFileBackend::QNetworkAccessFileBackend() - : totalBytes(0) + : uploadByteDevice(0), totalBytes(0), hasUploadFinished(false) { } @@ -126,6 +127,9 @@ void QNetworkAccessFileBackend::open() break; case QNetworkAccessManager::PutOperation: mode = QIODevice::WriteOnly | QIODevice::Truncate; + uploadByteDevice = createUploadByteDevice(); + QObject::connect(uploadByteDevice, SIGNAL(readyRead()), this, SLOT(uploadReadyReadSlot())); + QMetaObject::invokeMethod(this, "uploadReadyReadSlot", Qt::QueuedConnection); break; default: Q_ASSERT_X(false, "QNetworkAccessFileBackend::open", @@ -152,19 +156,50 @@ void QNetworkAccessFileBackend::open() } } -void QNetworkAccessFileBackend::closeDownstreamChannel() +void QNetworkAccessFileBackend::uploadReadyReadSlot() { - if (operation() == QNetworkAccessManager::GetOperation) { - file.close(); - //downstreamChannelClosed(); + if (hasUploadFinished) + return; + + forever { + qint64 haveRead; + const char *readPointer = uploadByteDevice->readPointer(-1, haveRead); + if (haveRead == -1) { + // EOF + hasUploadFinished = true; + file.flush(); + file.close(); + finished(); + break; + } else if (haveRead == 0 || readPointer == 0) { + // nothing to read right now, we will be called again later + break; + } else { + qint64 haveWritten; + haveWritten = file.write(readPointer, haveRead); + + if (haveWritten < 0) { + // write error! + QString msg = QCoreApplication::translate("QNetworkAccessFileBackend", "Write error writing to %1: %2") + .arg(url().toString(), file.errorString()); + error(QNetworkReply::ProtocolFailure, msg); + + finished(); + return; + } else { + uploadByteDevice->advanceReadPointer(haveWritten); + } + + + file.flush(); + } } } -void QNetworkAccessFileBackend::closeUpstreamChannel() +void QNetworkAccessFileBackend::closeDownstreamChannel() { - if (operation() == QNetworkAccessManager::PutOperation) { + if (operation() == QNetworkAccessManager::GetOperation) { file.close(); - finished(); } } @@ -174,40 +209,6 @@ bool QNetworkAccessFileBackend::waitForDownstreamReadyRead(int) return readMoreFromFile(); } -bool QNetworkAccessFileBackend::waitForUpstreamBytesWritten(int) -{ - Q_ASSERT_X(false, "QNetworkAccessFileBackend::waitForUpstreamBytesWritten", - "This function should never have been called, since there is never anything " - "left to be written!"); - return false; -} - -void QNetworkAccessFileBackend::upstreamReadyRead() -{ - Q_ASSERT_X(operation() == QNetworkAccessManager::PutOperation, "QNetworkAccessFileBackend", - "We're being told to upload data but operation isn't PUT!"); - - // there's more data to be written to the file - while (upstreamBytesAvailable()) { - // write everything and let QFile handle it - int written = file.write(readUpstream()); - - if (written < 0) { - // write error! - QString msg = QCoreApplication::translate("QNetworkAccessFileBackend", "Write error writing to %1: %2") - .arg(url().toString(), file.errorString()); - error(QNetworkReply::ProtocolFailure, msg); - - finished(); - return; - } - - // successful write - file.flush(); - upstreamBytesConsumed(written); - } -} - void QNetworkAccessFileBackend::downstreamReadyWrite() { Q_ASSERT_X(operation() == QNetworkAccessManager::GetOperation, "QNetworkAccessFileBackend", diff --git a/src/network/access/qnetworkaccessfilebackend_p.h b/src/network/access/qnetworkaccessfilebackend_p.h index ce7d351..4615c5f 100644 --- a/src/network/access/qnetworkaccessfilebackend_p.h +++ b/src/network/access/qnetworkaccessfilebackend_p.h @@ -62,22 +62,25 @@ QT_BEGIN_NAMESPACE class QNetworkAccessFileBackend: public QNetworkAccessBackend { + Q_OBJECT public: QNetworkAccessFileBackend(); virtual ~QNetworkAccessFileBackend(); virtual void open(); virtual void closeDownstreamChannel(); - virtual void closeUpstreamChannel(); virtual bool waitForDownstreamReadyRead(int msecs); - virtual bool waitForUpstreamBytesWritten(int msecs); - virtual void upstreamReadyRead(); virtual void downstreamReadyWrite(); +public slots: + void uploadReadyReadSlot(); +protected: + QNonContiguousByteDevice *uploadByteDevice; private: QFile file; qint64 totalBytes; + bool hasUploadFinished; bool loadFileInfo(); bool readMoreFromFile(); diff --git a/src/network/access/qnetworkaccessftpbackend.cpp b/src/network/access/qnetworkaccessftpbackend.cpp index ea39dec..ad55b85 100644 --- a/src/network/access/qnetworkaccessftpbackend.cpp +++ b/src/network/access/qnetworkaccessftpbackend.cpp @@ -42,6 +42,7 @@ #include "qnetworkaccessftpbackend_p.h" #include "qnetworkaccessmanager_p.h" #include "QtNetwork/qauthenticator.h" +#include "private/qnoncontiguousbytedevice_p.h" #ifndef QT_NO_FTP @@ -81,41 +82,6 @@ QNetworkAccessFtpBackendFactory::create(QNetworkAccessManager::Operation op, return 0; } -class QNetworkAccessFtpIODevice: public QIODevice -{ - //Q_OBJECT -public: - QNetworkAccessFtpBackend *backend; - bool eof; - - inline QNetworkAccessFtpIODevice(QNetworkAccessFtpBackend *parent) - : QIODevice(parent), backend(parent), eof(false) - { open(ReadOnly); } - - bool isSequential() const { return true; } - bool atEnd() const { return backend->upstreamBytesAvailable() == 0; } - - qint64 bytesAvailable() const { return backend->upstreamBytesAvailable(); } - qint64 bytesToWrite() const { return backend->downstreamBytesToConsume(); } -protected: - qint64 readData(char *data, qint64 maxlen) - { - const QByteArray toSend = backend->readUpstream(); - maxlen = qMin<qint64>(maxlen, toSend.size()); - if (!maxlen) - return eof ? -1 : 0; - - backend->upstreamBytesConsumed(maxlen); - memcpy(data, toSend.constData(), maxlen); - return maxlen; - } - - qint64 writeData(const char *, qint64) - { return -1; } - - friend class QNetworkAccessFtpBackend; -}; - class QNetworkAccessFtpFtp: public QFtp, public QNetworkAccessCache::CacheableObject { // Q_OBJECT @@ -198,7 +164,11 @@ void QNetworkAccessFtpBackend::open() ftpConnectionReady(ftp); } - uploadDevice = new QNetworkAccessFtpIODevice(this); + // Put operation + if (operation() == QNetworkAccessManager::PutOperation) { + uploadDevice = QNonContiguousByteDeviceFactory::wrap(createUploadByteDevice()); + uploadDevice->setParent(this); + } } void QNetworkAccessFtpBackend::closeDownstreamChannel() @@ -212,16 +182,6 @@ void QNetworkAccessFtpBackend::closeDownstreamChannel() #endif } -void QNetworkAccessFtpBackend::closeUpstreamChannel() -{ - if (operation() == QNetworkAccessManager::PutOperation) { - Q_ASSERT(uploadDevice); - uploadDevice->eof = true; - if (!upstreamBytesAvailable()) - emit uploadDevice->readyRead(); - } -} - bool QNetworkAccessFtpBackend::waitForDownstreamReadyRead(int ms) { if (!ftp) @@ -239,18 +199,6 @@ bool QNetworkAccessFtpBackend::waitForDownstreamReadyRead(int ms) return false; } -bool QNetworkAccessFtpBackend::waitForUpstreamBytesWritten(int ms) -{ - Q_UNUSED(ms); - qCritical("QNetworkAccess: FTP backend does not support waitForBytesWritten()"); - return false; -} - -void QNetworkAccessFtpBackend::upstreamReadyRead() -{ - // uh... how does QFtp operate? -} - void QNetworkAccessFtpBackend::downstreamReadyWrite() { if (state == Transferring && ftp && ftp->bytesAvailable()) diff --git a/src/network/access/qnetworkaccessftpbackend_p.h b/src/network/access/qnetworkaccessftpbackend_p.h index 9ec2dd8..1bb7ff2 100644 --- a/src/network/access/qnetworkaccessftpbackend_p.h +++ b/src/network/access/qnetworkaccessftpbackend_p.h @@ -87,11 +87,8 @@ public: virtual void open(); virtual void closeDownstreamChannel(); - virtual void closeUpstreamChannel(); virtual bool waitForDownstreamReadyRead(int msecs); - virtual bool waitForUpstreamBytesWritten(int msecs); - virtual void upstreamReadyRead(); virtual void downstreamReadyWrite(); void disconnectFromFtp(); @@ -105,7 +102,7 @@ public slots: private: friend class QNetworkAccessFtpIODevice; QPointer<QNetworkAccessFtpFtp> ftp; - QNetworkAccessFtpIODevice *uploadDevice; + QIODevice *uploadDevice; qint64 totalBytes; int helpId, sizeId, mdtmId; bool supportsSize, supportsMdtm; diff --git a/src/network/access/qnetworkaccesshttpbackend.cpp b/src/network/access/qnetworkaccesshttpbackend.cpp index a52b5a0..7d0438e 100644 --- a/src/network/access/qnetworkaccesshttpbackend.cpp +++ b/src/network/access/qnetworkaccesshttpbackend.cpp @@ -286,37 +286,6 @@ public: } }; -class QNetworkAccessHttpBackendIODevice: public QIODevice -{ - // Q_OBJECT -public: - bool eof; - QNetworkAccessHttpBackendIODevice(QNetworkAccessHttpBackend *parent) - : QIODevice(parent), eof(false) - { - setOpenMode(ReadOnly); - } - bool isSequential() const { return true; } - qint64 bytesAvailable() const - { return static_cast<QNetworkAccessHttpBackend *>(parent())->upstreamBytesAvailable(); } - -protected: - virtual qint64 readData(char *buffer, qint64 maxlen) - { - qint64 ret = static_cast<QNetworkAccessHttpBackend *>(parent())->deviceReadData(buffer, maxlen); - if (!ret && eof) - return -1; - return ret; - } - - virtual qint64 writeData(const char *, qint64) - { - return -1; // cannot write - } - - friend class QNetworkAccessHttpBackend; -}; - QNetworkAccessHttpBackend::QNetworkAccessHttpBackend() : QNetworkAccessBackend(), httpReply(0), http(0), uploadDevice(0) #ifndef QT_NO_OPENSSL @@ -507,20 +476,19 @@ void QNetworkAccessHttpBackend::postRequest() case QNetworkAccessManager::PostOperation: invalidateCache(); httpRequest.setOperation(QHttpNetworkRequest::Post); - uploadDevice = new QNetworkAccessHttpBackendIODevice(this); + httpRequest.setUploadByteDevice(createUploadByteDevice()); break; case QNetworkAccessManager::PutOperation: invalidateCache(); httpRequest.setOperation(QHttpNetworkRequest::Put); - uploadDevice = new QNetworkAccessHttpBackendIODevice(this); + httpRequest.setUploadByteDevice(createUploadByteDevice()); break; default: break; // can't happen } - httpRequest.setData(uploadDevice); httpRequest.setUrl(url()); QList<QByteArray> headers = request().rawHeaderList(); @@ -528,7 +496,9 @@ void QNetworkAccessHttpBackend::postRequest() httpRequest.setHeaderField(header, request().rawHeader(header)); if (loadedFromCache) { - QNetworkAccessBackend::finished(); + // commented this out since it will be called later anyway + // by copyFinished() + //QNetworkAccessBackend::finished(); return; // no need to send the request! :) } @@ -624,14 +594,6 @@ void QNetworkAccessHttpBackend::closeDownstreamChannel() // this indicates that the user closed the stream while the reply isn't finished yet } -void QNetworkAccessHttpBackend::closeUpstreamChannel() -{ - // this indicates that the user finished uploading the data for POST - Q_ASSERT(uploadDevice); - uploadDevice->eof = true; - emit uploadDevice->readChannelFinished(); -} - bool QNetworkAccessHttpBackend::waitForDownstreamReadyRead(int msecs) { Q_ASSERT(http); @@ -651,38 +613,6 @@ bool QNetworkAccessHttpBackend::waitForDownstreamReadyRead(int msecs) return false; } -bool QNetworkAccessHttpBackend::waitForUpstreamBytesWritten(int msecs) -{ - - // ### FIXME: not implemented in QHttpNetworkAccess - Q_UNUSED(msecs); - qCritical("QNetworkAccess: HTTP backend does not support waitForBytesWritten()"); - return false; -} - -void QNetworkAccessHttpBackend::upstreamReadyRead() -{ - // There is more data available from the user to be uploaded - // QHttpNetworkAccess implements the upload rate control: - // we simply tell QHttpNetworkAccess that there is more data available - // it'll pull from us when it can (through uploadDevice) - - Q_ASSERT(uploadDevice); - emit uploadDevice->readyRead(); -} - -qint64 QNetworkAccessHttpBackend::deviceReadData(char *buffer, qint64 maxlen) -{ - QByteArray toBeUploaded = readUpstream(); - if (toBeUploaded.isEmpty()) - return 0; // nothing to be uploaded - - maxlen = qMin<qint64>(maxlen, toBeUploaded.length()); - - memcpy(buffer, toBeUploaded.constData(), maxlen); - upstreamBytesConsumed(maxlen); - return maxlen; -} void QNetworkAccessHttpBackend::downstreamReadyWrite() { @@ -904,7 +834,14 @@ bool QNetworkAccessHttpBackend::sendCacheContents(const QNetworkCacheMetaData &m checkForRedirect(status); - writeDownstreamData(contents); + emit metaDataChanged(); + + // invoke this asynchronously, else Arora/QtDemoBrowser don't like cached downloads + // see task 250221 / 251801 + qRegisterMetaType<QIODevice*>("QIODevice*"); + QMetaObject::invokeMethod(this, "writeDownstreamData", Qt::QueuedConnection, Q_ARG(QIODevice*, contents)); + + #if defined(QNETWORKACCESSHTTPBACKEND_DEBUG) qDebug() << "Successfully sent cache:" << url() << contents->size() << "bytes"; #endif diff --git a/src/network/access/qnetworkaccesshttpbackend_p.h b/src/network/access/qnetworkaccesshttpbackend_p.h index 02915e7..225f944 100644 --- a/src/network/access/qnetworkaccesshttpbackend_p.h +++ b/src/network/access/qnetworkaccesshttpbackend_p.h @@ -79,11 +79,8 @@ public: virtual void open(); virtual void closeDownstreamChannel(); - virtual void closeUpstreamChannel(); virtual bool waitForDownstreamReadyRead(int msecs); - virtual bool waitForUpstreamBytesWritten(int msecs); - virtual void upstreamReadyRead(); virtual void downstreamReadyWrite(); virtual void copyFinished(QIODevice *); #ifndef QT_NO_OPENSSL @@ -96,6 +93,9 @@ public: qint64 deviceReadData(char *buffer, qint64 maxlen); + // we return true since HTTP needs to send PUT/POST data again after having authenticated + bool needsResetableUploadData() {return true;}; + private slots: void replyReadyRead(); void replyFinished(); @@ -108,7 +108,8 @@ private: QHttpNetworkReply *httpReply; QPointer<QNetworkAccessHttpBackendCache> http; QByteArray cacheKey; - QNetworkAccessHttpBackendIODevice *uploadDevice; + QNetworkAccessBackendUploadIODevice *uploadDevice; + #ifndef QT_NO_OPENSSL QSslConfiguration *pendingSslConfiguration; bool pendingIgnoreSslErrors; @@ -122,8 +123,6 @@ private: void postRequest(); void readFromHttp(); void checkForRedirect(const int statusCode); - - friend class QNetworkAccessHttpBackendIODevice; }; class QNetworkAccessHttpBackendFactory : public QNetworkAccessBackendFactory diff --git a/src/network/access/qnetworkaccessmanager.cpp b/src/network/access/qnetworkaccessmanager.cpp index bcbeef1..bf06ede 100644 --- a/src/network/access/qnetworkaccessmanager.cpp +++ b/src/network/access/qnetworkaccessmanager.cpp @@ -686,7 +686,10 @@ QNetworkReply *QNetworkAccessManager::createRequest(QNetworkAccessManager::Opera priv->urlForLastAuthentication = url; } - // third step: setup the reply + // third step: find a backend + priv->backend = d->findBackend(op, request); + + // fourth step: setup the reply priv->setup(op, request, outgoingData); if (request.attribute(QNetworkRequest::CacheLoadControlAttribute, QNetworkRequest::PreferNetwork).toInt() != QNetworkRequest::AlwaysNetwork) @@ -695,9 +698,6 @@ QNetworkReply *QNetworkAccessManager::createRequest(QNetworkAccessManager::Opera QList<QNetworkProxy> proxyList = d->queryProxy(QNetworkProxyQuery(request.url())); priv->proxyList = proxyList; #endif - - // fourth step: find a backend - priv->backend = d->findBackend(op, request); if (priv->backend) { priv->backend->setParent(reply); priv->backend->reply = priv; diff --git a/src/network/access/qnetworkreply.cpp b/src/network/access/qnetworkreply.cpp index f4dad3c..0990b17 100644 --- a/src/network/access/qnetworkreply.cpp +++ b/src/network/access/qnetworkreply.cpp @@ -151,6 +151,10 @@ QNetworkReplyPrivate::QNetworkReplyPrivate() authentication to serve the content but the credentials provided were not accepted (if any) + \value ContentReSendError the request needed to be sent + again, but this failed for example because the upload data + could not be read a second time. + \value ProtocolUnknownError the Network Access API cannot honor the request because the protocol is not known diff --git a/src/network/access/qnetworkreply.h b/src/network/access/qnetworkreply.h index 6f763b3..2f864fe 100644 --- a/src/network/access/qnetworkreply.h +++ b/src/network/access/qnetworkreply.h @@ -92,6 +92,7 @@ public: ContentOperationNotPermittedError, ContentNotFoundError, AuthenticationRequiredError, + ContentReSendError, UnknownContentError = 299, // protocol errors diff --git a/src/network/access/qnetworkreplyimpl.cpp b/src/network/access/qnetworkreplyimpl.cpp index 79c3d1a..749a462 100644 --- a/src/network/access/qnetworkreplyimpl.cpp +++ b/src/network/access/qnetworkreplyimpl.cpp @@ -46,13 +46,15 @@ #include "QtCore/qcoreapplication.h" #include "QtCore/qdatetime.h" #include "QtNetwork/qsslconfiguration.h" +#include "qnetworkaccesshttpbackend_p.h" #include <QtCore/QCoreApplication> QT_BEGIN_NAMESPACE inline QNetworkReplyImplPrivate::QNetworkReplyImplPrivate() - : copyDevice(0), networkCache(0), + : backend(0), outgoingData(0), outgoingDataBuffer(0), + copyDevice(0), networkCache(0), cacheEnabled(false), cacheSaveDevice(0), bytesDownloaded(0), lastBytesDownloaded(-1), bytesUploaded(-1), state(Idle) @@ -61,8 +63,13 @@ inline QNetworkReplyImplPrivate::QNetworkReplyImplPrivate() void QNetworkReplyImplPrivate::_q_startOperation() { - // This function is called exactly once + // ensure this function is only being called once + if (state == Working) { + qDebug("QNetworkReplyImpl::_q_startOperation was called more than once"); + return; + } state = Working; + if (!backend) { error(QNetworkReplyImpl::ProtocolUnknownError, QCoreApplication::translate("QNetworkReply", "Protocol \"%1\" is unknown").arg(url.scheme())); // not really true!; @@ -74,57 +81,11 @@ void QNetworkReplyImplPrivate::_q_startOperation() if (state != Finished) { if (operation == QNetworkAccessManager::GetOperation) pendingNotifications.append(NotifyDownstreamReadyWrite); - if (outgoingData) { - _q_sourceReadyRead(); -#if 0 // ### FIXME - if (outgoingData->atEndOfStream() && writeBuffer.isEmpty()) - // empty upload - emit q->uploadProgress(0, 0); -#endif - } handleNotifications(); } } -void QNetworkReplyImplPrivate::_q_sourceReadyRead() -{ - // read data from the outgoingData QIODevice into our internal buffer - enum { DesiredBufferSize = 32 * 1024 }; - - if (writeBuffer.size() >= DesiredBufferSize) - return; // don't grow the buffer too much - - // read as many bytes are available or up until we fill up the buffer - // but always read at least one byte - qint64 bytesToRead = qBound<qint64>(1, outgoingData->bytesAvailable(), - DesiredBufferSize - writeBuffer.size()); - char *ptr = writeBuffer.reserve(bytesToRead); - qint64 bytesActuallyRead = outgoingData->read(ptr, bytesToRead); - if (bytesActuallyRead == -1) { - // EOF - writeBuffer.chop(bytesToRead); - backendNotify(NotifyCloseUpstreamChannel); - return; - } - - if (bytesActuallyRead < bytesToRead) - writeBuffer.chop(bytesToRead - bytesActuallyRead); - - // if we did read anything, let the backend know and handle it - if (bytesActuallyRead) - backendNotify(NotifyUpstreamReadyRead); - - // check for EOF again - if (!outgoingData->isSequential() && outgoingData->atEnd()) - backendNotify(NotifyCloseUpstreamChannel); -} - -void QNetworkReplyImplPrivate::_q_sourceReadChannelFinished() -{ - _q_sourceReadyRead(); -} - void QNetworkReplyImplPrivate::_q_copyReadyRead() { Q_Q(QNetworkReplyImpl); @@ -143,7 +104,7 @@ void QNetworkReplyImplPrivate::_q_copyReadyRead() if (bytesActuallyRead == -1) { readBuffer.chop(bytesToRead); backendNotify(NotifyCopyFinished); - return; + break; } if (bytesActuallyRead != bytesToRead) @@ -151,6 +112,7 @@ void QNetworkReplyImplPrivate::_q_copyReadyRead() if (!copyDevice->isSequential() && copyDevice->atEnd()) { backendNotify(NotifyCopyFinished); + bytesDownloaded += bytesActuallyRead; break; } @@ -174,6 +136,67 @@ void QNetworkReplyImplPrivate::_q_copyReadChannelFinished() _q_copyReadyRead(); } +void QNetworkReplyImplPrivate::_q_bufferOutgoingDataFinished() +{ + Q_Q(QNetworkReplyImpl); + + // make sure this is only called once, ever. + //_q_bufferOutgoingData may call it or the readChannelFinished emission + if (state != Buffering) + return; + + // disconnect signals + QObject::disconnect(outgoingData, SIGNAL(readyRead()), q, SLOT(_q_bufferOutgoingData())); + QObject::disconnect(outgoingData, SIGNAL(readChannelFinished()), q, SLOT(_q_bufferOutgoingDataFinished())); + + // finally, start the request + QMetaObject::invokeMethod(q, "_q_startOperation", Qt::QueuedConnection); +} + +void QNetworkReplyImplPrivate::_q_bufferOutgoingData() +{ + Q_Q(QNetworkReplyImpl); + + if (!outgoingDataBuffer) { + // first call, create our buffer + outgoingDataBuffer = new QRingBuffer(); + + QObject::connect(outgoingData, SIGNAL(readyRead()), q, SLOT(_q_bufferOutgoingData())); + QObject::connect(outgoingData, SIGNAL(readChannelFinished()), q, SLOT(_q_bufferOutgoingDataFinished())); + } + + qint64 bytesBuffered = 0; + qint64 bytesToBuffer = 0; + + // read data into our buffer + forever { + bytesToBuffer = outgoingData->bytesAvailable(); + // unknown? just try 2 kB, this also ensures we always try to read the EOF + if (bytesToBuffer <= 0) + bytesToBuffer = 2*1024; + + char *dst = outgoingDataBuffer->reserve(bytesToBuffer); + bytesBuffered = outgoingData->read(dst, bytesToBuffer); + + if (bytesBuffered == -1) { + // EOF has been reached. + outgoingDataBuffer->chop(bytesToBuffer); + + _q_bufferOutgoingDataFinished(); + break; + } else if (bytesBuffered == 0) { + // nothing read right now, just wait until we get called again + outgoingDataBuffer->chop(bytesToBuffer); + + break; + } else { + // don't break, try to read() again + outgoingDataBuffer->chop(bytesToBuffer - bytesBuffered); + } + } +} + + void QNetworkReplyImplPrivate::setup(QNetworkAccessManager::Operation op, const QNetworkRequest &req, QIODevice *data) { @@ -184,13 +207,42 @@ void QNetworkReplyImplPrivate::setup(QNetworkAccessManager::Operation op, const url = request.url(); operation = op; - if (outgoingData) { - q->connect(outgoingData, SIGNAL(readyRead()), SLOT(_q_sourceReadyRead())); - q->connect(outgoingData, SIGNAL(readChannelFinished()), SLOT(_q_sourceReadChannelFinished())); + if (outgoingData && backend) { + // there is data to be uploaded, e.g. HTTP POST. + + if (!backend->needsResetableUploadData() || !outgoingData->isSequential()) { + // backend does not need upload buffering or + // fixed size non-sequential + // just start the operation + QMetaObject::invokeMethod(q, "_q_startOperation", Qt::QueuedConnection); + } else { + bool bufferingDisallowed = + req.attribute(QNetworkRequest::DoNotBufferUploadDataAttribute, + false).toBool(); + + if (bufferingDisallowed) { + // if a valid content-length header for the request was supplied, we can disable buffering + // if not, we will buffer anyway + if (req.header(QNetworkRequest::ContentLengthHeader).isValid()) { + QMetaObject::invokeMethod(q, "_q_startOperation", Qt::QueuedConnection); + } else { + state = Buffering; + QMetaObject::invokeMethod(q, "_q_bufferOutgoingData", Qt::QueuedConnection); + } + } else { + // _q_startOperation will be called when the buffering has finished. + state = Buffering; + QMetaObject::invokeMethod(q, "_q_bufferOutgoingData", Qt::QueuedConnection); + } + } + } else { + // No outgoing data (e.g. HTTP GET request) + // or no backend + // if no backend, _q_startOperation will handle the error of this + QMetaObject::invokeMethod(q, "_q_startOperation", Qt::QueuedConnection); } q->QIODevice::open(QIODevice::ReadOnly); - QMetaObject::invokeMethod(q, "_q_startOperation", Qt::QueuedConnection); } void QNetworkReplyImplPrivate::setNetworkCache(QAbstractNetworkCache *nc) @@ -226,18 +278,10 @@ void QNetworkReplyImplPrivate::handleNotifications() backend->downstreamReadyWrite(); break; - case NotifyUpstreamReadyRead: - backend->upstreamReadyRead(); - break; - case NotifyCloseDownstreamChannel: backend->closeDownstreamChannel(); break; - case NotifyCloseUpstreamChannel: - backend->closeUpstreamChannel(); - break; - case NotifyCopyFinished: { QIODevice *dev = copyDevice; copyDevice = 0; @@ -299,29 +343,14 @@ void QNetworkReplyImplPrivate::completeCacheSave() cacheEnabled = false; } -void QNetworkReplyImplPrivate::consume(qint64 count) +void QNetworkReplyImplPrivate::emitUploadProgress(qint64 bytesSent, qint64 bytesTotal) { Q_Q(QNetworkReplyImpl); - if (count <= 0) { - qWarning("QNetworkConnection: backend signalled that it consumed %ld bytes", long(count)); - return; - } - - if (outgoingData) - // schedule another read from the source - QMetaObject::invokeMethod(q_func(), "_q_sourceReadyRead", Qt::QueuedConnection); - - writeBuffer.skip(count); - if (bytesUploaded == -1) - bytesUploaded = count; - else - bytesUploaded += count; - - QVariant totalSize = request.header(QNetworkRequest::ContentLengthHeader); - emit q->uploadProgress(bytesUploaded, - totalSize.isNull() ? Q_INT64_C(-1) : totalSize.toLongLong()); + bytesUploaded = bytesSent; + emit q->uploadProgress(bytesSent, bytesTotal); } + qint64 QNetworkReplyImplPrivate::nextDownstreamBlockSize() const { enum { DesiredBufferSize = 32 * 1024 }; @@ -331,7 +360,9 @@ qint64 QNetworkReplyImplPrivate::nextDownstreamBlockSize() const return qMax<qint64>(0, readBufferMaxSize - readBuffer.size()); } -void QNetworkReplyImplPrivate::feed(const QByteArray &data) +// we received downstream data and send this to the cache +// and to our readBuffer (which in turn gets read by the user of QNetworkReply) +void QNetworkReplyImplPrivate::appendDownstreamData(const QByteArray &data) { Q_Q(QNetworkReplyImpl); if (!q->isOpen()) @@ -379,7 +410,8 @@ void QNetworkReplyImplPrivate::feed(const QByteArray &data) } } -void QNetworkReplyImplPrivate::feed(QIODevice *data) +// this is used when it was fetched from the cache, right? +void QNetworkReplyImplPrivate::appendDownstreamData(QIODevice *data) { Q_Q(QNetworkReplyImpl); Q_ASSERT(q->isOpen()); @@ -409,9 +441,11 @@ void QNetworkReplyImplPrivate::finished() pendingNotifications.clear(); QVariant totalSize = cookedHeaders.value(QNetworkRequest::ContentLengthHeader); - if (bytesDownloaded != lastBytesDownloaded || totalSize.isNull()) + if (totalSize.isNull() || totalSize == -1) { emit q->downloadProgress(bytesDownloaded, bytesDownloaded); - if (bytesUploaded == -1 && outgoingData) + } + + if (bytesUploaded == -1 && (outgoingData || outgoingDataBuffer)) emit q->uploadProgress(0, 0); completeCacheSave(); diff --git a/src/network/access/qnetworkreplyimpl_p.h b/src/network/access/qnetworkreplyimpl_p.h index ad06f78..8d3c90e 100644 --- a/src/network/access/qnetworkreplyimpl_p.h +++ b/src/network/access/qnetworkreplyimpl_p.h @@ -59,6 +59,7 @@ #include "qnetworkproxy.h" #include "QtCore/qmap.h" #include "QtCore/qqueue.h" +#include "QtCore/qbuffer.h" #include "private/qringbuffer_p.h" QT_BEGIN_NAMESPACE @@ -91,10 +92,10 @@ public: Q_DECLARE_PRIVATE(QNetworkReplyImpl) Q_PRIVATE_SLOT(d_func(), void _q_startOperation()) - Q_PRIVATE_SLOT(d_func(), void _q_sourceReadyRead()) - Q_PRIVATE_SLOT(d_func(), void _q_sourceReadChannelFinished()) Q_PRIVATE_SLOT(d_func(), void _q_copyReadyRead()) Q_PRIVATE_SLOT(d_func(), void _q_copyReadChannelFinished()) + Q_PRIVATE_SLOT(d_func(), void _q_bufferOutgoingData()) + Q_PRIVATE_SLOT(d_func(), void _q_bufferOutgoingDataFinished()) }; class QNetworkReplyImplPrivate: public QNetworkReplyPrivate @@ -102,15 +103,13 @@ class QNetworkReplyImplPrivate: public QNetworkReplyPrivate public: enum InternalNotifications { NotifyDownstreamReadyWrite, - NotifyUpstreamReadyRead, NotifyCloseDownstreamChannel, - NotifyCloseUpstreamChannel, NotifyCopyFinished }; enum State { Idle, - Opening, + Buffering, Working, Finished, Aborted @@ -125,6 +124,8 @@ public: void _q_sourceReadChannelFinished(); void _q_copyReadyRead(); void _q_copyReadChannelFinished(); + void _q_bufferOutgoingData(); + void _q_bufferOutgoingDataFinished(); void setup(QNetworkAccessManager::Operation op, const QNetworkRequest &request, QIODevice *outgoingData); @@ -138,9 +139,10 @@ public: void setCachingEnabled(bool enable); bool isCachingEnabled() const; void consume(qint64 count); + void emitUploadProgress(qint64 bytesSent, qint64 bytesTotal); qint64 nextDownstreamBlockSize() const; - void feed(const QByteArray &data); - void feed(QIODevice *data); + void appendDownstreamData(const QByteArray &data); + void appendDownstreamData(QIODevice *data); void finished(); void error(QNetworkReply::NetworkError code, const QString &errorString); void metaDataChanged(); @@ -149,6 +151,7 @@ public: QNetworkAccessBackend *backend; QIODevice *outgoingData; + QRingBuffer *outgoingDataBuffer; QIODevice *copyDevice; QAbstractNetworkCache *networkCache; diff --git a/src/network/access/qnetworkrequest.cpp b/src/network/access/qnetworkrequest.cpp index 56b793d..8b1afba 100644 --- a/src/network/access/qnetworkrequest.cpp +++ b/src/network/access/qnetworkrequest.cpp @@ -162,6 +162,13 @@ QT_BEGIN_NAMESPACE Indicates whether the data was obtained from cache or not. + \value DoNotBufferUploadDataAttribute + Requests only, type: QVariant::Bool (default: false) + Indicates whether the QNetworkAccessManager code is + allowed to buffer the upload data, e.g. when doing a HTTP POST. + When using this flag with sequential upload data, the ContentLengthHeader + header must be set. + \value User Special type. Additional information can be passed in QVariants with types ranging from User to UserMax. The default diff --git a/src/network/access/qnetworkrequest.h b/src/network/access/qnetworkrequest.h index 6f34bce..5dea1df 100644 --- a/src/network/access/qnetworkrequest.h +++ b/src/network/access/qnetworkrequest.h @@ -75,6 +75,7 @@ public: CacheLoadControlAttribute, CacheSaveControlAttribute, SourceIsFromCacheAttribute, + DoNotBufferUploadDataAttribute, User = 1000, UserMax = 32767 |