diff options
author | Markus Goetz <Markus.Goetz@nokia.com> | 2009-05-13 11:12:25 (GMT) |
---|---|---|
committer | Markus Goetz <Markus.Goetz@nokia.com> | 2009-05-13 11:42:04 (GMT) |
commit | 9d827648b7618cd3bdad6548e83b1d16453c96bf (patch) | |
tree | ba9fa348a8fa2edb66a4b7ddd3120d99554d9db7 /src | |
parent | 66534102f76c89c6f07f42dab0a47b0d2debec97 (diff) | |
download | Qt-9d827648b7618cd3bdad6548e83b1d16453c96bf.zip Qt-9d827648b7618cd3bdad6548e83b1d16453c96bf.tar.gz Qt-9d827648b7618cd3bdad6548e83b1d16453c96bf.tar.bz2 |
QNAM: Upload architecture change: HTTP implementation
Reviewed-by: Peter Hartmann
Reviewed-by: Thiago Macieira
Diffstat (limited to 'src')
-rw-r--r-- | src/network/access/qhttpnetworkconnection.cpp | 215 | ||||
-rw-r--r-- | src/network/access/qhttpnetworkconnection_p.h | 8 | ||||
-rw-r--r-- | src/network/access/qhttpnetworkreply.cpp | 12 | ||||
-rw-r--r-- | src/network/access/qhttpnetworkreply_p.h | 7 | ||||
-rw-r--r-- | src/network/access/qhttpnetworkrequest.cpp | 19 | ||||
-rw-r--r-- | src/network/access/qhttpnetworkrequest_p.h | 9 |
6 files changed, 144 insertions, 126 deletions
diff --git a/src/network/access/qhttpnetworkconnection.cpp b/src/network/access/qhttpnetworkconnection.cpp index 5940fba..af0ac84 100644 --- a/src/network/access/qhttpnetworkconnection.cpp +++ b/src/network/access/qhttpnetworkconnection.cpp @@ -40,6 +40,7 @@ ****************************************************************************/ #include "qhttpnetworkconnection_p.h" +#include "private/qnoncontiguousbytedevice_p.h" #include <private/qnetworkrequest_p.h> #include <private/qobject_p.h> #include <private/qauthenticator_p.h> @@ -71,6 +72,7 @@ QHttpNetworkConnectionPrivate::QHttpNetworkConnectionPrivate(const QString &host #ifndef QT_NO_NETWORKPROXY , networkProxy(QNetworkProxy::NoProxy) #endif + { } @@ -205,12 +207,19 @@ void QHttpNetworkConnectionPrivate::prepareRequest(HttpMessagePair &messagePair) // add missing fields for the request QByteArray value; // check if Content-Length is provided - QIODevice *data = request.data(); - if (data && request.contentLength() == -1) { - if (!data->isSequential()) - request.setContentLength(data->size()); - else - bufferData(messagePair); // ### or do chunked upload + QNonContiguousByteDevice* uploadByteDevice = request.uploadByteDevice(); + if (uploadByteDevice) { + if (request.contentLength() != -1 && uploadByteDevice->size() != -1) { + // both values known, take the smaller one. + request.setContentLength(qMin(uploadByteDevice->size(), request.contentLength())); + } else if (request.contentLength() == -1 && uploadByteDevice->size() != -1) { + // content length not supplied by user, but the upload device knows it + request.setContentLength(uploadByteDevice->size()); + } else if (request.contentLength() != -1 && uploadByteDevice->size() == -1) { + // everything OK, the user supplied us the contentLength + } else if (request.contentLength() == -1 && uploadByteDevice->size() == -1) { + qFatal("QHttpNetworkConnectionPrivate: Neither content-length nor upload device size were given"); + } } // set the Connection/Proxy-Connection: Keep-Alive headers #ifndef QT_NO_NETWORKPROXY @@ -361,18 +370,12 @@ bool QHttpNetworkConnectionPrivate::sendRequest(QAbstractSocket *socket) false); #endif socket->write(header); - QIODevice *data = channels[i].request.d->data; - QHttpNetworkReply *reply = channels[i].reply; - if (reply && reply->d_func()->requestDataBuffer.size()) - data = &channels[i].reply->d_func()->requestDataBuffer; - if (data && (data->isOpen() || data->open(QIODevice::ReadOnly))) { - if (data->isSequential()) { - channels[i].bytesTotal = -1; - QObject::connect(data, SIGNAL(readyRead()), q, SLOT(_q_dataReadyReadNoBuffer())); - QObject::connect(data, SIGNAL(readChannelFinished()), q, SLOT(_q_dataReadyReadNoBuffer())); - } else { - channels[i].bytesTotal = data->size(); - } + QNonContiguousByteDevice* uploadByteDevice = channels[i].request.uploadByteDevice(); + if (uploadByteDevice) { + // connect the signals so this function gets called again + QObject::connect(uploadByteDevice, SIGNAL(readyRead()), q, SLOT(_q_uploadDataReadyRead())); + + channels[i].bytesTotal = channels[i].request.contentLength(); } else { channels[i].state = WaitingState; break; @@ -380,30 +383,81 @@ bool QHttpNetworkConnectionPrivate::sendRequest(QAbstractSocket *socket) // write the initial chunk together with the headers // fall through } - case WritingState: { // write the data - QIODevice *data = channels[i].request.d->data; - if (channels[i].reply->d_func()->requestDataBuffer.size()) - data = &channels[i].reply->d_func()->requestDataBuffer; - if (!data || channels[i].bytesTotal == channels[i].written) { + case WritingState: + { + // write the data + QNonContiguousByteDevice* uploadByteDevice = channels[i].request.uploadByteDevice(); + if (!uploadByteDevice || channels[i].bytesTotal == channels[i].written) { + if (uploadByteDevice) + emit channels[i].reply->dataSendProgress(channels[i].written, channels[i].bytesTotal); channels[i].state = WaitingState; // now wait for response + sendRequest(socket); break; } - QByteArray chunk; - chunk.resize(ChunkSize); - qint64 readSize = data->read(chunk.data(), ChunkSize); - if (readSize == -1) { - // source has reached EOF - channels[i].state = WaitingState; // now wait for response - } else if (readSize > 0) { - // source gave us something useful - channels[i].written += socket->write(chunk.data(), readSize); - if (channels[i].reply) - emit channels[i].reply->dataSendProgress(channels[i].written, channels[i].bytesTotal); + // only feed the QTcpSocket buffer when there is less than 32 kB in it + const qint64 socketBufferFill = 32*1024; + const qint64 socketWriteMaxSize = 16*1024; + + +#ifndef QT_NO_OPENSSL + QSslSocket *sslSocket = qobject_cast<QSslSocket*>(socket); + while ((sslSocket->encryptedBytesToWrite() + sslSocket->bytesToWrite()) <= socketBufferFill + && channels[i].bytesTotal != channels[i].written) +#else + while (socket->bytesToWrite() <= socketBufferFill + && channels[i].bytesTotal != channels[i].written) +#endif + { + // get pointer to upload data + qint64 currentReadSize; + qint64 desiredReadSize = qMin(socketWriteMaxSize, channels[i].bytesTotal - channels[i].written); + const char *readPointer = uploadByteDevice->readPointer(desiredReadSize, currentReadSize); + + if (currentReadSize == -1) { + // premature eof happened + emitReplyError(socket, channels[i].reply, QNetworkReply::UnknownNetworkError); + return false; + break; + } else if (readPointer == 0 || currentReadSize == 0) { + // nothing to read currently, break the loop + break; + } else { + qint64 currentWriteSize = socket->write(readPointer, currentReadSize); + if (currentWriteSize == -1 || currentWriteSize != currentReadSize) { + // socket broke down + emitReplyError(socket, channels[i].reply, QNetworkReply::UnknownNetworkError); + return false; + } else { + channels[i].written += currentWriteSize; + uploadByteDevice->advanceReadPointer(currentWriteSize); + + emit channels[i].reply->dataSendProgress(channels[i].written, channels[i].bytesTotal); + + if (channels[i].written == channels[i].bytesTotal) { + // make sure this function is called once again + channels[i].state = WaitingState; + sendRequest(socket); + break; + } + } + } } break; } + case WaitingState: + { + QNonContiguousByteDevice* uploadByteDevice = channels[i].request.uploadByteDevice(); + if (uploadByteDevice) { + QObject::disconnect(uploadByteDevice, SIGNAL(readyRead()), q, SLOT(_q_uploadDataReadyRead())); + } + // ensure we try to receive a reply in all cases, even if _q_readyRead_ hat not been called + // this is needed if the sends an reply before we have finished sending the request. In that + // case receiveReply had been called before but ignored the server reply + receiveReply(socket, channels[i].reply); + break; + } case ReadingState: case Wait4AuthState: // ignore _q_bytesWritten in these states @@ -479,6 +533,9 @@ bool QHttpNetworkConnectionPrivate::expand(QAbstractSocket *socket, QHttpNetwork // make sure that the reply is valid if (channels[i].reply != reply) return true; + // emit dataReadProgress signal (signal is currently not connected + // to the rest of QNAM) since readProgress of the + // QNonContiguousByteDevice is used emit reply->dataReadProgress(reply->d_func()->totalProgress, 0); // make sure that the reply is valid if (channels[i].reply != reply) @@ -569,6 +626,9 @@ void QHttpNetworkConnectionPrivate::receiveReply(QAbstractSocket *socket, QHttpN // make sure that the reply is valid if (channels[i].reply != reply) return; + // emit dataReadProgress signal (signal is currently not connected + // to the rest of QNAM) since readProgress of the + // QNonContiguousByteDevice is used emit reply->dataReadProgress(reply->d_func()->totalProgress, reply->d_func()->bodyLength); // make sure that the reply is valid if (channels[i].reply != reply) @@ -635,8 +695,25 @@ void QHttpNetworkConnectionPrivate::handleStatus(QAbstractSocket *socket, QHttpN case 407: handleAuthenticateChallenge(socket, reply, (statusCode == 407), resend); if (resend) { + int i = indexOf(socket); + + QNonContiguousByteDevice* uploadByteDevice = channels[i].request.uploadByteDevice(); + if (uploadByteDevice) { + if (uploadByteDevice->reset()) { + channels[i].written = 0; + } else { + emitReplyError(socket, reply, QNetworkReply::ContentReSendError); + break; + } + } + eraseData(reply); - sendRequest(socket); + + // also use async _q_startNextRequest so we dont break with closed + // proxy or server connections.. + channels[i].resendCurrent = true; + QMetaObject::invokeMethod(q, "_q_startNextRequest", Qt::QueuedConnection); + } break; default: @@ -970,6 +1047,7 @@ void QHttpNetworkConnectionPrivate::_q_bytesWritten(qint64 bytes) QAbstractSocket *socket = qobject_cast<QAbstractSocket*>(q->sender()); if (!socket) return; // ### error + // bytes have been written to the socket. write even more of them :) if (isSocketWriting(socket)) sendRequest(socket); // otherwise we do nothing @@ -1128,80 +1206,21 @@ void QHttpNetworkConnectionPrivate::_q_proxyAuthenticationRequired(const QNetwor } #endif -void QHttpNetworkConnectionPrivate::_q_dataReadyReadNoBuffer() +void QHttpNetworkConnectionPrivate::_q_uploadDataReadyRead() { Q_Q(QHttpNetworkConnection); - // data emitted either readyRead() + // upload data emitted readyRead() // find out which channel it is for - QIODevice *sender = qobject_cast<QIODevice *>(q->sender()); + QObject *sender = q->sender(); - // won't match anything if the qobject_cast above failed for (int i = 0; i < channelCount; ++i) { - if (sender == channels[i].request.data()) { + if (sender == channels[i].request.uploadByteDevice()) { sendRequest(channels[i].socket); break; } } } -void QHttpNetworkConnectionPrivate::_q_dataReadyReadBuffer() -{ - Q_Q(QHttpNetworkConnection); - QIODevice *sender = qobject_cast<QIODevice *>(q->sender()); - HttpMessagePair *thePair = 0; - for (int i = 0; !thePair && i < lowPriorityQueue.size(); ++i) - if (lowPriorityQueue.at(i).first.data() == sender) - thePair = &lowPriorityQueue[i]; - - for (int i = 0; !thePair && i < highPriorityQueue.size(); ++i) - if (highPriorityQueue.at(i).first.data() == sender) - thePair = &highPriorityQueue[i]; - - if (thePair) { - bufferData(*thePair); - - // are we finished buffering? - if (!thePair->second->d_func()->requestIsBuffering) - _q_startNextRequest(); - } -} - -void QHttpNetworkConnectionPrivate::bufferData(HttpMessagePair &messagePair) -{ - Q_Q(QHttpNetworkConnection); - QHttpNetworkRequest &request = messagePair.first; - QHttpNetworkReply *reply = messagePair.second; - Q_ASSERT(request.data()); - if (!reply->d_func()->requestIsBuffering) { // first time - QObject::connect(request.data(), SIGNAL(readyRead()), q, SLOT(_q_dataReadyReadBuffer())); - QObject::connect(request.data(), SIGNAL(readChannelFinished()), q, SLOT(_q_dataReadyReadBuffer())); - reply->d_func()->requestIsBuffering = true; - reply->d_func()->requestDataBuffer.open(QIODevice::ReadWrite); - } - - // always try to read at least one byte - // ### FIXME! use a QRingBuffer - qint64 bytesToRead = qMax<qint64>(1, request.data()->bytesAvailable()); - QByteArray newData; - newData.resize(bytesToRead); - qint64 bytesActuallyRead = request.data()->read(newData.data(), bytesToRead); - - if (bytesActuallyRead > 0) { - // we read something - newData.chop(bytesToRead - bytesActuallyRead); - reply->d_func()->requestDataBuffer.write(newData); - } else if (bytesActuallyRead == -1) { // last time - QObject::disconnect(request.data(), SIGNAL(readyRead()), q, SLOT(_q_dataReadyReadBuffer())); - QObject::disconnect(request.data(), SIGNAL(readChannelFinished()), q, SLOT(_q_dataReadyReadBuffer())); - - request.setContentLength(reply->d_func()->requestDataBuffer.size()); - reply->d_func()->requestDataBuffer.seek(0); - reply->d_func()->requestIsBuffering = false; - } -} - -// QHttpNetworkConnection - QHttpNetworkConnection::QHttpNetworkConnection(const QString &hostName, quint16 port, bool encrypt, QObject *parent) : QObject(*(new QHttpNetworkConnectionPrivate(hostName, port, encrypt)), parent) { diff --git a/src/network/access/qhttpnetworkconnection_p.h b/src/network/access/qhttpnetworkconnection_p.h index 09bd459..9b127dd 100644 --- a/src/network/access/qhttpnetworkconnection_p.h +++ b/src/network/access/qhttpnetworkconnection_p.h @@ -146,8 +146,7 @@ private: #ifndef QT_NO_NETWORKPROXY Q_PRIVATE_SLOT(d_func(), void _q_proxyAuthenticationRequired(const QNetworkProxy&, QAuthenticator*)) #endif - Q_PRIVATE_SLOT(d_func(), void _q_dataReadyReadBuffer()) - Q_PRIVATE_SLOT(d_func(), void _q_dataReadyReadNoBuffer()) + Q_PRIVATE_SLOT(d_func(), void _q_uploadDataReadyRead()) #ifndef QT_NO_OPENSSL Q_PRIVATE_SLOT(d_func(), void _q_encrypted()) @@ -209,8 +208,8 @@ public: #ifndef QT_NO_NETWORKPROXY void _q_proxyAuthenticationRequired(const QNetworkProxy &proxy, QAuthenticator *auth); // from transparent proxy #endif - void _q_dataReadyReadNoBuffer(); - void _q_dataReadyReadBuffer(); + + void _q_uploadDataReadyRead(); void createAuthorization(QAbstractSocket *socket, QHttpNetworkRequest &request); bool ensureConnection(QAbstractSocket *socket); @@ -219,7 +218,6 @@ public: #ifndef QT_NO_COMPRESS bool expand(QAbstractSocket *socket, QHttpNetworkReply *reply, bool dataComplete); #endif - void bufferData(HttpMessagePair &request); void removeReply(QHttpNetworkReply *reply); QString hostName; diff --git a/src/network/access/qhttpnetworkreply.cpp b/src/network/access/qhttpnetworkreply.cpp index fe3f6af..4485485 100644 --- a/src/network/access/qhttpnetworkreply.cpp +++ b/src/network/access/qhttpnetworkreply.cpp @@ -521,13 +521,13 @@ qint64 QHttpNetworkReplyPrivate::readBody(QAbstractSocket *socket, QIODevice *ou { qint64 bytes = 0; if (isChunked()) { - bytes += transferChunked(socket, out); // chunked transfer encoding (rfc 2616, sec 3.6) + bytes += readReplyBodyChunked(socket, out); // chunked transfer encoding (rfc 2616, sec 3.6) } else if (bodyLength > 0) { // we have a Content-Length - bytes += transferRaw(socket, out, bodyLength - contentRead); + bytes += readReplyBodyRaw(socket, out, bodyLength - contentRead); if (contentRead + bytes == bodyLength) state = AllDoneState; } else { - bytes += transferRaw(socket, out, socket->bytesAvailable()); + bytes += readReplyBodyRaw(socket, out, socket->bytesAvailable()); } if (state == AllDoneState) socket->readAll(); // Read the rest to clean (CRLF) @@ -535,7 +535,7 @@ qint64 QHttpNetworkReplyPrivate::readBody(QAbstractSocket *socket, QIODevice *ou return bytes; } -qint64 QHttpNetworkReplyPrivate::transferRaw(QIODevice *in, QIODevice *out, qint64 size) +qint64 QHttpNetworkReplyPrivate::readReplyBodyRaw(QIODevice *in, QIODevice *out, qint64 size) { qint64 bytes = 0; Q_ASSERT(in); @@ -561,7 +561,7 @@ qint64 QHttpNetworkReplyPrivate::transferRaw(QIODevice *in, QIODevice *out, qint } -qint64 QHttpNetworkReplyPrivate::transferChunked(QIODevice *in, QIODevice *out) +qint64 QHttpNetworkReplyPrivate::readReplyBodyChunked(QIODevice *in, QIODevice *out) { qint64 bytes = 0; while (in->bytesAvailable()) { // while we can read from input @@ -660,4 +660,4 @@ void QHttpNetworkReply::ignoreSslErrors() QT_END_NAMESPACE -#endif // QT_NO_HTTP
\ No newline at end of file +#endif // QT_NO_HTTP diff --git a/src/network/access/qhttpnetworkreply_p.h b/src/network/access/qhttpnetworkreply_p.h index c17c65c..238a1da 100644 --- a/src/network/access/qhttpnetworkreply_p.h +++ b/src/network/access/qhttpnetworkreply_p.h @@ -139,7 +139,7 @@ Q_SIGNALS: void finishedWithError(QNetworkReply::NetworkError errorCode, const QString &detail = QString()); void headerChanged(); void dataReadProgress(int done, int total); - void dataSendProgress(int done, int total); + void dataSendProgress(qint64 done, qint64 total); private: Q_DECLARE_PRIVATE(QHttpNetworkReply) @@ -162,8 +162,8 @@ public: QAuthenticatorPrivate::Method authenticationMethod(bool isProxy) const; void clear(); - qint64 transferRaw(QIODevice *in, QIODevice *out, qint64 size); - qint64 transferChunked(QIODevice *in, QIODevice *out); + qint64 readReplyBodyRaw(QIODevice *in, QIODevice *out, qint64 size); + qint64 readReplyBodyChunked(QIODevice *in, QIODevice *out); qint64 getChunkSize(QIODevice *in, qint64 *chunkSize); qint64 bytesAvailable() const; @@ -206,7 +206,6 @@ public: QByteArray responseData; // uncompressed body QByteArray compressedData; // compressed body (temporary) - QBuffer requestDataBuffer; bool requestIsBuffering; bool requestIsPrepared; }; diff --git a/src/network/access/qhttpnetworkrequest.cpp b/src/network/access/qhttpnetworkrequest.cpp index 420cb69..7df68fc 100644 --- a/src/network/access/qhttpnetworkrequest.cpp +++ b/src/network/access/qhttpnetworkrequest.cpp @@ -40,12 +40,13 @@ ****************************************************************************/ #include "qhttpnetworkrequest_p.h" +#include "private/qnoncontiguousbytedevice_p.h" QT_BEGIN_NAMESPACE QHttpNetworkRequestPrivate::QHttpNetworkRequestPrivate(QHttpNetworkRequest::Operation op, QHttpNetworkRequest::Priority pri, const QUrl &newUrl) - : QHttpNetworkHeaderPrivate(newUrl), operation(op), priority(pri), data(0), + : QHttpNetworkHeaderPrivate(newUrl), operation(op), priority(pri), uploadByteDevice(0), autoDecompress(false) { } @@ -55,7 +56,7 @@ QHttpNetworkRequestPrivate::QHttpNetworkRequestPrivate(const QHttpNetworkRequest { operation = other.operation; priority = other.priority; - data = other.data; + uploadByteDevice = other.uploadByteDevice; autoDecompress = other.autoDecompress; } @@ -67,7 +68,7 @@ bool QHttpNetworkRequestPrivate::operator==(const QHttpNetworkRequestPrivate &ot { return QHttpNetworkHeaderPrivate::operator==(other) && (operation == other.operation) - && (data == other.data); + && (uploadByteDevice == other.uploadByteDevice); } QByteArray QHttpNetworkRequestPrivate::methodName() const @@ -109,7 +110,7 @@ QByteArray QHttpNetworkRequestPrivate::uri(bool throughProxy) const QUrl::FormattingOptions format(QUrl::RemoveFragment); // for POST, query data is send as content - if (operation == QHttpNetworkRequest::Post && !data) + if (operation == QHttpNetworkRequest::Post && !uploadByteDevice) format |= QUrl::RemoveQuery; // for requests through proxy, the Request-URI contains full url if (throughProxy) @@ -140,7 +141,7 @@ QByteArray QHttpNetworkRequestPrivate::header(const QHttpNetworkRequest &request // add content type, if not set in the request if (request.headerField("content-type").isEmpty()) ba += "Content-Type: application/x-www-form-urlencoded\r\n"; - if (!request.d->data && request.d->url.hasQuery()) { + if (!request.d->uploadByteDevice && request.d->url.hasQuery()) { QByteArray query = request.d->url.encodedQuery(); ba += "Content-Length: "+ QByteArray::number(query.size()) + "\r\n"; ba += "\r\n"; @@ -236,14 +237,14 @@ void QHttpNetworkRequest::setPriority(Priority priority) d->priority = priority; } -QIODevice *QHttpNetworkRequest::data() const +void QHttpNetworkRequest::setUploadByteDevice(QNonContiguousByteDevice *bd) { - return d->data; + d->uploadByteDevice = bd; } -void QHttpNetworkRequest::setData(QIODevice *data) +QNonContiguousByteDevice* QHttpNetworkRequest::uploadByteDevice() const { - d->data = data; + return d->uploadByteDevice; } int QHttpNetworkRequest::majorVersion() const diff --git a/src/network/access/qhttpnetworkrequest_p.h b/src/network/access/qhttpnetworkrequest_p.h index d18e116..ed4325a 100644 --- a/src/network/access/qhttpnetworkrequest_p.h +++ b/src/network/access/qhttpnetworkrequest_p.h @@ -58,6 +58,8 @@ QT_BEGIN_NAMESPACE +class QNonContiguousByteDevice; + class QHttpNetworkRequestPrivate; class Q_AUTOTEST_EXPORT QHttpNetworkRequest: public QHttpNetworkHeader { @@ -104,8 +106,8 @@ public: Priority priority() const; void setPriority(Priority priority); - QIODevice *data() const; - void setData(QIODevice *data); + void setUploadByteDevice(QNonContiguousByteDevice *bd); + QNonContiguousByteDevice* uploadByteDevice() const; private: QSharedDataPointer<QHttpNetworkRequestPrivate> d; @@ -113,7 +115,6 @@ private: friend class QHttpNetworkConnectionPrivate; }; - class QHttpNetworkRequestPrivate : public QHttpNetworkHeaderPrivate { public: @@ -129,7 +130,7 @@ public: QHttpNetworkRequest::Operation operation; QHttpNetworkRequest::Priority priority; - mutable QIODevice *data; + mutable QNonContiguousByteDevice* uploadByteDevice; bool autoDecompress; }; |