QNAM HTTP: Fix readBuffer maximum size.

QHttpNetworkReply has now a throttled mode. If this is activated,
it will only read HTTP body when its internal buffer is empty.
This means that QNetworkReply.setReadBufferSize() can finally
be used for bandwidth limiting.
Thanks to David Faure for helping out.

Reviewed-By: Peter Hartmann

12 files changed, 197 insertions, 1 deletions

diff --git a/src/network/access/qhttpnetworkconnection.cpp b/src/network/access/qhttpnetworkconnection.cpp
index 2c56524..b7dbeac 100644
--- a/src/network/access/qhttpnetworkconnection.cpp
+++ b/src/network/access/qhttpnetworkconnection.cpp
@@ -731,6 +731,16 @@ void QHttpNetworkConnectionPrivate::_q_restartAuthPendingRequests()
     }
 }
 
+void QHttpNetworkConnectionPrivate::readMoreLater(QHttpNetworkReply *reply)
+{
+    for (int i = 0 ; i < channelCount; ++i) {
+        if (channels[i].reply ==  reply) {
+            // emulate a readyRead() from the socket
+            QMetaObject::invokeMethod(&channels[i], "_q_readyRead", Qt::QueuedConnection);
+            return;
+        }
+    }
+}
 
 QHttpNetworkConnection::QHttpNetworkConnection(const QString &hostName, quint16 port, bool encrypt, QObject *parent)
     : QObject(*(new QHttpNetworkConnectionPrivate(hostName, port, encrypt)), parent)
diff --git a/src/network/access/qhttpnetworkconnection_p.h b/src/network/access/qhttpnetworkconnection_p.h
index 00eb1ef..76da883 100644
--- a/src/network/access/qhttpnetworkconnection_p.h
+++ b/src/network/access/qhttpnetworkconnection_p.h
@@ -174,6 +174,9 @@ public:
     void fillPipeline(QAbstractSocket *socket);
     bool fillPipeline(QList<HttpMessagePair> &queue, QHttpNetworkConnectionChannel &channel);
 
+    // read more HTTP body after the next event loop spin
+    void readMoreLater(QHttpNetworkReply *reply);
+
     void copyCredentials(int fromChannel, QAuthenticator *auth, bool isProxy);
 
     // private slots
diff --git a/src/network/access/qhttpnetworkconnectionchannel.cpp b/src/network/access/qhttpnetworkconnectionchannel.cpp
index 53c2596..b1b0304 100644
--- a/src/network/access/qhttpnetworkconnectionchannel.cpp
+++ b/src/network/access/qhttpnetworkconnectionchannel.cpp
@@ -350,6 +350,16 @@ void QHttpNetworkConnectionChannel::_q_receiveReply()
             }
             break;
         case QHttpNetworkReplyPrivate::ReadingDataState: {
+           if (reply->d_func()->downstreamLimited && !reply->d_func()->responseData.isEmpty() && reply->d_func()->shouldEmitSignals()) {
+               // We already have some HTTP body data. We don't read more from the socket until
+               // this is fetched by QHttpNetworkAccessHttpBackend. If we would read more,
+               // we could not limit our read buffer usage.
+               // We only do this when shouldEmitSignals==true because our HTTP parsing
+               // always needs to parse the 401/407 replies. Therefore they don't really obey
+               // to the read buffer maximum size, but we don't care since they should be small.
+               return;
+           }
+
             if (!reply->d_func()->isChunked() && !reply->d_func()->autoDecompress
                 && reply->d_func()->bodyLength > 0) {
                 // bulk files like images should fulfill these properties and
diff --git a/src/network/access/qhttpnetworkreply.cpp b/src/network/access/qhttpnetworkreply.cpp
index 7025f1d..a5223d1 100644
--- a/src/network/access/qhttpnetworkreply.cpp
+++ b/src/network/access/qhttpnetworkreply.cpp
@@ -179,9 +179,19 @@ qint64 QHttpNetworkReply::bytesAvailableNextBlock() const
 QByteArray QHttpNetworkReply::readAny()
 {
     Q_D(QHttpNetworkReply);
+    // we'll take the last buffer, so schedule another read from http
+    if (d->downstreamLimited && d->responseData.bufferCount() == 1)
+        d->connection->d_func()->readMoreLater(this);
     return d->responseData.read();
 }
 
+void QHttpNetworkReply::setDownstreamLimited(bool dsl)
+{
+    Q_D(QHttpNetworkReply);
+    d->downstreamLimited = dsl;
+    d->connection->d_func()->readMoreLater(this);
+}
+
 bool QHttpNetworkReply::isFinished() const
 {
     return d_func()->state == QHttpNetworkReplyPrivate::AllDoneState;
@@ -201,7 +211,7 @@ QHttpNetworkReplyPrivate::QHttpNetworkReplyPrivate(const QUrl &newUrl)
       forceConnectionCloseEnabled(false),
       currentChunkSize(0), currentChunkRead(0), connection(0), initInflate(false),
       autoDecompress(false), responseData(), requestIsPrepared(false)
-      ,pipeliningUsed(false)
+      ,pipeliningUsed(false), downstreamLimited(false)
 {
 }
 
diff --git a/src/network/access/qhttpnetworkreply_p.h b/src/network/access/qhttpnetworkreply_p.h
index a8b4a79..af9266b 100644
--- a/src/network/access/qhttpnetworkreply_p.h
+++ b/src/network/access/qhttpnetworkreply_p.h
@@ -125,6 +125,7 @@ public:
     qint64 bytesAvailable() const;
     qint64 bytesAvailableNextBlock() const;
     QByteArray readAny();
+    void setDownstreamLimited(bool t);
 
     bool isFinished() const;
 
@@ -229,6 +230,7 @@ public:
     bool requestIsPrepared;
 
     bool pipeliningUsed;
+    bool downstreamLimited;
 };
 
 
diff --git a/src/network/access/qnetworkaccessbackend.cpp b/src/network/access/qnetworkaccessbackend.cpp
index de947e8..8ac64d2 100644
--- a/src/network/access/qnetworkaccessbackend.cpp
+++ b/src/network/access/qnetworkaccessbackend.cpp
@@ -161,6 +161,12 @@ void QNetworkAccessBackend::downstreamReadyWrite()
     // do nothing
 }
 
+void QNetworkAccessBackend::setDownstreamLimited(bool b)
+{
+    Q_UNUSED(b);
+    // do nothing
+}
+
 void QNetworkAccessBackend::copyFinished(QIODevice *)
 {
     // do nothing
diff --git a/src/network/access/qnetworkaccessbackend_p.h b/src/network/access/qnetworkaccessbackend_p.h
index 30ef5b3..43d993c 100644
--- a/src/network/access/qnetworkaccessbackend_p.h
+++ b/src/network/access/qnetworkaccessbackend_p.h
@@ -116,6 +116,7 @@ public:
 
     // slot-like:
     virtual void downstreamReadyWrite();
+    virtual void setDownstreamLimited(bool b);
     virtual void copyFinished(QIODevice *);
     virtual void ignoreSslErrors();
     virtual void ignoreSslErrors(const QList<QSslError> &errors);
diff --git a/src/network/access/qnetworkaccesshttpbackend.cpp b/src/network/access/qnetworkaccesshttpbackend.cpp
index dacb2bb..a639e0d 100644
--- a/src/network/access/qnetworkaccesshttpbackend.cpp
+++ b/src/network/access/qnetworkaccesshttpbackend.cpp
@@ -657,6 +657,12 @@ void QNetworkAccessHttpBackend::downstreamReadyWrite()
         replyFinished();
 }
 
+void QNetworkAccessHttpBackend::setDownstreamLimited(bool b)
+{
+    if (httpReply)
+        httpReply->setDownstreamLimited(b);
+}
+
 void QNetworkAccessHttpBackend::replyReadyRead()
 {
     readFromHttp();
diff --git a/src/network/access/qnetworkaccesshttpbackend_p.h b/src/network/access/qnetworkaccesshttpbackend_p.h
index 705323d..0eaf003 100644
--- a/src/network/access/qnetworkaccesshttpbackend_p.h
+++ b/src/network/access/qnetworkaccesshttpbackend_p.h
@@ -82,6 +82,8 @@ public:
     virtual bool waitForDownstreamReadyRead(int msecs);
 
     virtual void downstreamReadyWrite();
+    virtual void setDownstreamLimited(bool b);
+
     virtual void copyFinished(QIODevice *);
 #ifndef QT_NO_OPENSSL
     virtual void ignoreSslErrors();
diff --git a/src/network/access/qnetworkreplyimpl.cpp b/src/network/access/qnetworkreplyimpl.cpp
index 285864d..59c7d76 100644
--- a/src/network/access/qnetworkreplyimpl.cpp
+++ b/src/network/access/qnetworkreplyimpl.cpp
@@ -652,6 +652,9 @@ void QNetworkReplyImpl::setReadBufferSize(qint64 size)
         d->backendNotify(QNetworkReplyImplPrivate::NotifyDownstreamReadyWrite);
 
     QNetworkReply::setReadBufferSize(size);
+
+    if (d->backend)
+        d->backend->setDownstreamLimited(d->readBufferMaxSize > 0);
 }
 
 #ifndef QT_NO_OPENSSL
diff --git a/tests/manual/qnetworkreply/main.cpp b/tests/manual/qnetworkreply/main.cpp
new file mode 100644
index 0000000..015d66e
--- /dev/null
+++ b/tests/manual/qnetworkreply/main.cpp
@@ -0,0 +1,130 @@
+/****************************************************************************
+**
+** Copyright (C) 2010 Nokia Corporation and/or its subsidiary(-ies).
+** All rights reserved.
+** Contact: Nokia Corporation (qt-info@nokia.com)
+**
+** This file is part of the test suite of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:LGPL$
+** No Commercial Usage
+** This file contains pre-release code and may not be distributed.
+** You may use this file in accordance with the terms and conditions
+** contained in the Technology Preview License Agreement accompanying
+** this package.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 2.1 as published by the Free Software
+** Foundation and appearing in the file LICENSE.LGPL included in the
+** packaging of this file.  Please review the following information to
+** ensure the GNU Lesser General Public License version 2.1 requirements
+** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+**
+** In addition, as a special exception, Nokia gives you certain additional
+** rights.  These rights are described in the Nokia Qt LGPL Exception
+** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
+**
+** If you have questions regarding the use of this file, please contact
+** Nokia at qt-info@nokia.com.
+**
+**
+**
+**
+**
+**
+**
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+// This file contains benchmarks for QNetworkReply functions.
+
+#include <QDebug>
+#include <qtest.h>
+#include <QtTest/QtTest>
+#include <QtNetwork/qnetworkreply.h>
+#include <QtNetwork/qnetworkrequest.h>
+#include <QtNetwork/qnetworkaccessmanager.h>
+#include "../../auto/network-settings.h"
+
+#define BANDWIDTH_LIMIT_BYTES (1024*100)
+#define TIME_ESTIMATION_SECONDS (97)
+
+class tst_qnetworkreply : public QObject
+{
+    Q_OBJECT
+    private slots:
+    void limiting_data();
+    void limiting();
+
+};
+
+QNetworkReply *reply;
+
+class HttpReceiver : public QObject
+{
+    Q_OBJECT
+    public slots:
+    void finishedSlot() {
+        quint64 bytesPerSec = (reply->header(QNetworkRequest::ContentLengthHeader).toLongLong()) / (stopwatch.elapsed() / 1000.0);
+        qDebug() << "Finished HTTP(S) request with" << bytesPerSec << "bytes/sec";
+        QVERIFY(bytesPerSec < BANDWIDTH_LIMIT_BYTES*1.05);
+        QVERIFY(bytesPerSec > BANDWIDTH_LIMIT_BYTES*0.95);
+        timer->stop();
+        QTestEventLoop::instance().exitLoop();
+    }
+    void readyReadSlot() {
+    }
+    void timeoutSlot() {
+        reply->read(BANDWIDTH_LIMIT_BYTES).size();
+    }
+    void startTimer() {
+        stopwatch.start();
+        timer = new QTimer(this);
+        QObject::connect(timer, SIGNAL(timeout()), this, SLOT(timeoutSlot()));
+        timer->start(1000);
+    }
+protected:
+    QTimer *timer;
+    QTime stopwatch;
+};
+
+void tst_qnetworkreply::limiting_data()
+{
+    QTest::addColumn<QUrl>("url");
+
+    QTest::newRow("HTTP") << QUrl("http://" + QtNetworkSettings::serverName() + "/mediumfile");
+#ifndef QT_NO_OPENSSL
+    QTest::newRow("HTTP+SSL") << QUrl("https://" + QtNetworkSettings::serverName() + "/mediumfile");
+#endif
+
+}
+
+void tst_qnetworkreply::limiting()
+{
+    HttpReceiver receiver;
+    QNetworkAccessManager manager;
+
+    QFETCH(QUrl, url);
+    QNetworkRequest req (url);
+
+    qDebug() << "Starting. This will take a while (around" << TIME_ESTIMATION_SECONDS << "sec).";
+    qDebug() << "Please check the actual bandwidth usage with a network monitor, e.g. the KDE";
+    qDebug() << "network plasma widget. It should be around" << BANDWIDTH_LIMIT_BYTES << "bytes/sec.";
+    reply = manager.get(req);
+    reply->ignoreSslErrors();
+    reply->setReadBufferSize(BANDWIDTH_LIMIT_BYTES);
+    QObject::connect(reply, SIGNAL(readyRead()), &receiver, SLOT(readyReadSlot()));
+    QObject::connect(reply, SIGNAL(finished()), &receiver, SLOT(finishedSlot()));
+    receiver.startTimer();
+
+    // event loop
+    QTestEventLoop::instance().enterLoop(TIME_ESTIMATION_SECONDS + 20);
+    QVERIFY(!QTestEventLoop::instance().timeout());
+}
+
+
+QTEST_MAIN(tst_qnetworkreply)
+
+#include "main.moc"
diff --git a/tests/manual/qnetworkreply/qnetworkreply.pro b/tests/manual/qnetworkreply/qnetworkreply.pro
new file mode 100644
index 0000000..060acf5
--- /dev/null
+++ b/tests/manual/qnetworkreply/qnetworkreply.pro
@@ -0,0 +1,13 @@
+load(qttest_p4)
+TEMPLATE = app
+TARGET = tst_qnetworkreply
+DEPENDPATH += .
+INCLUDEPATH += .
+
+QT -= gui
+QT += network
+
+CONFIG += release
+
+# Input
+SOURCES += main.cpp